From 98ea9bd6a904762f7d6486d50964db0398a1b400 Mon Sep 17 00:00:00 2001 From: Percy Liang Date: Sun, 21 Dec 2014 18:43:16 -0800 Subject: [PATCH] Pre-release of SEMPRE 2.0 --- .gitignore | 9 - DOCUMENTATION.md | 1237 +++++++++++++++++ Makefile | 43 +- QUICKSTART.md | 93 -- README.md | 157 ++- TUTORIAL.md | 743 ++++++---- data/tutorial-arithmetic.examples | 4 + ...al.grammar => tutorial-arithmetic.grammar} | 12 +- data/tutorial.examples.json | 11 - data/unittest-learn.examples.json | 4 - demo-www/bullet.png | Bin 0 -> 966 bytes demo-www/index.html | 1 + demo-www/main.css | 64 + demo-www/main.js | 2 + download-dependencies | 70 - fig/bin/chunk | 4 +- fig/bin/qcreate | 5 +- freebase/README.md | 3 + freebase/data/emnlp2013.grammar | 127 ++ .../free917.test.examples.canonicalized.json | 278 ++++ .../free917.train.examples.canonicalized.json | 643 +++++++++ freebase/data/tutorial-freebase.grammar | 6 + {data => freebase/data}/tutorial.ttl | 2 +- .../data}/unittest-learn-ccg.grammar | 0 freebase/data/unittest-learn.examples | 9 + .../data}/unittest-learn.grammar | 0 .../scripts}/extract-freebase-schema.rb | 11 +- freebase/scripts/fbshell.rb | 210 +++ {scripts => freebase/scripts}/virtuoso | 37 +- module-classes.txt | 361 +++++ parasempre | 178 --- pull-dependencies | 163 +++ release-acl2014.files | 46 - release-code.files | 244 ---- release-core.files | 47 - release-emnlp2013.files | 40 - release-fullfreebase.files | 9 - release-fullfreebase_ttl.files | 9 - release-fullfreebase_vdb.files | 9 - release-geofreebase.files | 10 - release-geofreebase_ttl.files | 8 - release-geofreebase_vdb.files | 8 - run | 426 ++++++ scripts/agenda-stats | 7 + scripts/checkstyle.sh | 10 + scripts/checkstyle.xml | 197 +++ scripts/evaluation.py | 67 + scripts/extract-module-classes.rb | 24 + scripts/find-first-pred-diff.sh | 10 + scripts/find-hard-coded-paths.rb | 19 + scripts/fix-checkstyle.rb | 113 ++ scripts/tunnel | 38 + scripts/verify-code-loop.rb | 107 ++ sempre | 432 ------ .../AbstractReinforcementParserState.java | 203 +++ .../stanford/nlp/sempre/AggregateFormula.java | 20 +- .../nlp/sempre/ArithmeticFormula.java | 85 ++ .../stanford/nlp/sempre/AtomicSemType.java | 31 + .../nlp/sempre/BadFormulaException.java | 21 + src/edu/stanford/nlp/sempre/BeamParser.java | 279 ++-- src/edu/stanford/nlp/sempre/BooleanValue.java | 9 +- .../nlp/sempre/BoundedPriorityQueue.java | 77 + src/edu/stanford/nlp/sempre/BridgeFn.java | 409 ------ src/edu/stanford/nlp/sempre/Builder.java | 44 +- src/edu/stanford/nlp/sempre/CallFormula.java | 14 +- src/edu/stanford/nlp/sempre/CallTypeInfo.java | 16 + .../stanford/nlp/sempre/CanonicalNames.java | 42 + .../stanford/nlp/sempre/ChartParserState.java | 122 ++ src/edu/stanford/nlp/sempre/CoarseParser.java | 284 ++++ src/edu/stanford/nlp/sempre/Colorizer.java | 50 + src/edu/stanford/nlp/sempre/ConcatFn.java | 45 +- src/edu/stanford/nlp/sempre/ConstantFn.java | 56 +- src/edu/stanford/nlp/sempre/ContextFn.java | 123 ++ src/edu/stanford/nlp/sempre/ContextValue.java | 144 ++ src/edu/stanford/nlp/sempre/Dataset.java | 71 +- src/edu/stanford/nlp/sempre/DateFn.java | 46 +- src/edu/stanford/nlp/sempre/DateValue.java | 38 +- .../sempre/DerivOpCountFeatureComputer.java | 58 + .../sempre/DerivOpCountFeatureExtractor.java | 81 -- src/edu/stanford/nlp/sempre/Derivation.java | 337 ++--- .../nlp/sempre/DerivationConstraint.java | 28 - .../stanford/nlp/sempre/DerivationStream.java | 13 + .../stanford/nlp/sempre/DescriptionValue.java | 21 +- src/edu/stanford/nlp/sempre/ErrorValue.java | 8 +- src/edu/stanford/nlp/sempre/Evaluation.java | 130 -- .../nlp/sempre/ExactValueEvaluator.java | 8 + src/edu/stanford/nlp/sempre/Example.java | 201 +-- src/edu/stanford/nlp/sempre/ExampleUtils.java | 63 + src/edu/stanford/nlp/sempre/Executor.java | 6 +- .../stanford/nlp/sempre/FeatureComputer.java | 25 + .../stanford/nlp/sempre/FeatureExtractor.java | 309 +++- .../stanford/nlp/sempre/FeatureMatcher.java | 8 +- .../stanford/nlp/sempre/FeatureVector.java | 107 +- .../stanford/nlp/sempre/FilterNerSpanFn.java | 43 +- .../stanford/nlp/sempre/FilterPosTagFn.java | 88 +- .../nlp/sempre/FilterSpanLengthFn.java | 48 +- .../nlp/sempre/FloatingFeatureComputer.java | 65 + .../stanford/nlp/sempre/FloatingParser.java | 357 +++++ .../nlp/sempre/FloatingRuleUtils.java | 44 + src/edu/stanford/nlp/sempre/Formula.java | 18 +- .../nlp/sempre/FormulaGenerationInfo.java | 119 -- .../nlp/sempre/FormulaMatchExecutor.java | 13 +- .../stanford/nlp/sempre/FormulaRetriever.java | 431 ------ src/edu/stanford/nlp/sempre/Formulas.java | 212 ++- src/edu/stanford/nlp/sempre/FreebaseInfo.java | 364 ----- src/edu/stanford/nlp/sempre/FuncSemType.java | 47 + src/edu/stanford/nlp/sempre/FuzzyMatchFn.java | 102 ++ src/edu/stanford/nlp/sempre/Grammar.java | 202 ++- src/edu/stanford/nlp/sempre/HasScore.java | 8 + src/edu/stanford/nlp/sempre/IdentityFn.java | 27 +- src/edu/stanford/nlp/sempre/JavaExecutor.java | 265 +++- src/edu/stanford/nlp/sempre/JoinFn.java | 324 +++-- src/edu/stanford/nlp/sempre/JoinFormula.java | 14 +- src/edu/stanford/nlp/sempre/Json.java | 20 +- .../stanford/nlp/sempre/KnowledgeGraph.java | 107 ++ .../stanford/nlp/sempre/LambdaFormula.java | 12 +- .../stanford/nlp/sempre/LanguageAnalyzer.java | 31 + src/edu/stanford/nlp/sempre/LanguageInfo.java | 310 ++--- src/edu/stanford/nlp/sempre/Learner.java | 330 ++--- src/edu/stanford/nlp/sempre/LexiconFn.java | 334 ----- src/edu/stanford/nlp/sempre/ListValue.java | 45 +- src/edu/stanford/nlp/sempre/Main.java | 7 + src/edu/stanford/nlp/sempre/MarkFormula.java | 12 +- src/edu/stanford/nlp/sempre/Master.java | 275 ++-- src/edu/stanford/nlp/sempre/MergeFn.java | 102 +- src/edu/stanford/nlp/sempre/MergeFormula.java | 14 +- .../nlp/sempre/MultipleDerivationStream.java | 48 + .../nlp/sempre/NaiveKnowledgeGraph.java | 192 +++ src/edu/stanford/nlp/sempre/NameValue.java | 8 +- src/edu/stanford/nlp/sempre/NotFormula.java | 12 +- src/edu/stanford/nlp/sempre/NullExecutor.java | 2 +- .../stanford/nlp/sempre/NullTypeLookup.java | 17 + src/edu/stanford/nlp/sempre/NumberFn.java | 107 +- src/edu/stanford/nlp/sempre/NumberValue.java | 8 +- src/edu/stanford/nlp/sempre/Params.java | 132 +- .../stanford/nlp/sempre/ParaphraseModel.java | 75 + src/edu/stanford/nlp/sempre/Parser.java | 508 ++----- src/edu/stanford/nlp/sempre/ParserAgenda.java | 120 ++ src/edu/stanford/nlp/sempre/ParserState.java | 407 +++--- src/edu/stanford/nlp/sempre/Parsers.java | 11 - .../stanford/nlp/sempre/PrimitiveFormula.java | 5 + .../nlp/sempre/ReinforcementParser.java | 973 +++++++++++++ .../nlp/sempre/ReinforcementUtils.java | 148 ++ .../stanford/nlp/sempre/ReverseFormula.java | 18 +- src/edu/stanford/nlp/sempre/Rule.java | 98 +- src/edu/stanford/nlp/sempre/SelectFn.java | 57 +- src/edu/stanford/nlp/sempre/SemType.java | 226 +-- .../stanford/nlp/sempre/SemTypeHierarchy.java | 82 ++ src/edu/stanford/nlp/sempre/SemanticFn.java | 60 +- src/edu/stanford/nlp/sempre/SempreUtils.java | 34 + src/edu/stanford/nlp/sempre/Server.java | 678 +++++++++ src/edu/stanford/nlp/sempre/Session.java | 63 +- .../stanford/nlp/sempre/SimpleAnalyzer.java | 86 ++ .../stanford/nlp/sempre/SimpleLexicon.java | 123 ++ .../stanford/nlp/sempre/SimpleLexiconFn.java | 107 ++ .../nlp/sempre/SingleDerivationStream.java | 43 + src/edu/stanford/nlp/sempre/StringCache.java | 136 -- src/edu/stanford/nlp/sempre/StringValue.java | 14 +- .../nlp/sempre/SuperlativeFormula.java | 46 +- src/edu/stanford/nlp/sempre/TableValue.java | 81 ++ src/edu/stanford/nlp/sempre/TopSemType.java | 12 + .../stanford/nlp/sempre/TypeInference.java | 251 ++++ src/edu/stanford/nlp/sempre/TypeLookup.java | 17 + src/edu/stanford/nlp/sempre/UnionSemType.java | 64 + src/edu/stanford/nlp/sempre/UriValue.java | 12 +- src/edu/stanford/nlp/sempre/Value.java | 12 +- .../stanford/nlp/sempre/ValueEvaluator.java | 10 + src/edu/stanford/nlp/sempre/ValueFormula.java | 2 +- src/edu/stanford/nlp/sempre/Values.java | 42 +- .../stanford/nlp/sempre/VariableFormula.java | 4 +- src/edu/stanford/nlp/sempre/Vis.java | 233 ---- src/edu/stanford/nlp/sempre/build.xml | 17 + .../nlp/sempre/cache/FileStringCache.java | 168 +++ .../nlp/sempre/cache/LruCallback.java | 10 + src/edu/stanford/nlp/sempre/cache/LruMap.java | 86 ++ .../nlp/sempre/cache/RemoteStringCache.java | 82 ++ .../nlp/sempre/cache/StringCache.java | 12 + .../sempre/{ => cache}/StringCacheServer.java | 27 +- .../sempre/{ => cache}/StringCacheUtils.java | 16 +- src/edu/stanford/nlp/sempre/cache/build.xml | 14 + .../sempre/cache/test/StringCacheTest.java | 112 ++ .../nlp/sempre/corenlp/CoreNLPAnalyzer.java | 136 ++ src/edu/stanford/nlp/sempre/corenlp/build.xml | 14 + .../corenlp/test/CoreNLPSemanticFnTest.java | 92 ++ .../fbalignment/lexicons/BinaryLexicon.java | 348 ----- .../fbalignment/lexicons/UnaryLexicon.java | 308 ---- .../fbalignment/lexicons/WordDistance.java | 77 - .../lexicons/normalizers/EntryNormalizer.java | 7 - .../fbalignment/utils/CollectionUtils.java | 25 - .../nlp/sempre/freebase/BinaryLexicon.java | 282 ++++ .../nlp/sempre/freebase/BridgeFn.java | 548 ++++++++ .../sempre/freebase/BuildCanonicalIdMap.java | 146 ++ .../nlp/sempre/freebase/BuildTypesMap.java | 64 + .../sempre/freebase/CanonicalizeExamples.java | 83 ++ .../nlp/sempre/freebase/CanonicalizeIds.java | 80 ++ .../lexicons => freebase}/EntityLexicon.java | 110 +- .../nlp/sempre/freebase/ExecuteExamples.java | 84 ++ .../sempre/{ => freebase}/FbFormulasInfo.java | 337 ++--- .../nlp/sempre/freebase/FilterFreebase.java | 218 +++ .../nlp/sempre/freebase/Free917Converter.java | 727 ++++++++++ .../nlp/sempre/freebase/FreebaseInfo.java | 293 ++++ .../sempre/{ => freebase}/FreebaseSearch.java | 36 +- .../sempre/freebase/FreebaseTypeLookup.java | 64 + .../freebase/FreebaseValueEvaluator.java | 92 ++ .../freebase/LambdaCalculusConverter.java | 544 ++++++++ .../lexicons => freebase}/Lexicon.java | 46 +- .../nlp/sempre/freebase/LexiconFn.java | 412 ++++++ .../sempre/{ => freebase}/SparqlExecutor.java | 552 +++++--- .../nlp/sempre/{ => freebase}/SparqlExpr.java | 78 +- .../nlp/sempre/{ => freebase}/Stemmer.java | 2 +- .../{ => freebase}/TextToTextMatcher.java | 76 +- .../nlp/sempre/freebase/UnaryLexicon.java | 193 +++ .../stanford/nlp/sempre/freebase/Utils.java | 22 +- .../stanford/nlp/sempre/freebase/build.xml | 14 + .../index/FbEntityIndexer.java | 9 +- .../index/FbEntitySearcher.java | 16 +- .../index/FbIndexField.java | 4 +- .../lexicons/EntrySource.java | 7 +- .../lexicons/ExtremeValueWrapper.java | 2 +- .../lexicons/LexicalEntry.java | 109 +- .../lexicons/TokenLevelMatchFeatures.java | 9 +- .../normalizers/BinaryNormalizer.java | 2 +- .../lexicons/normalizers/EntryNormalizer.java | 5 + .../normalizers/IdentityNormalizer.java | 2 +- .../normalizers/PrepDropNormalizer.java | 2 +- .../{ => freebase}/test/FbFormulasTest.java | 32 +- .../freebase/test/FreebaseInfoTest.java | 24 + .../freebase/test/FreebaseSemTypeTest.java | 67 + .../test/FreebaseTypeInferenceTest.java | 92 ++ .../nlp/sempre/freebase/test/LexiconTest.java | 85 ++ .../test/PrepDropNormalizerTest.java | 12 +- .../test/SparqlExecutorTest.java | 37 +- .../{ => freebase}/test/StemmerTest.java | 4 +- .../{ => freebase}/test/TokenMatchTest.java | 4 +- .../freebase/utils/CollectionUtils.java | 25 + .../utils/DoubleContainer.java | 4 +- .../utils/FileUtils.java | 17 +- .../freebase/utils/FormatConverter.java | 82 ++ .../sempre/freebase/utils/FreebaseUtils.java | 180 +++ .../utils/LinkedExtractionFileUtils.java | 110 ++ .../utils/MathUtils.java | 103 +- .../freebase/utils/SemparseLogTools.java | 261 ++++ .../sempre/freebase/utils/ShortContainer.java | 28 + .../utils/WnExpander.java | 12 +- .../utils/WordNet.java | 178 ++- .../nlp/sempre/paraphrase/Aligner.java | 383 ----- .../nlp/sempre/paraphrase/Context.java | 98 -- .../nlp/sempre/paraphrase/ContextModel.java | 314 ----- .../paraphrase/ContextSimilarityModel.java | 149 -- .../nlp/sempre/paraphrase/DatasetUtils.java | 29 - .../nlp/sempre/paraphrase/EntityInstance.java | 30 - .../nlp/sempre/paraphrase/EntityModel.java | 104 -- .../sempre/paraphrase/FeatureSimilarity.java | 46 - .../paraphrase/FeatureSimilarityComputer.java | 106 -- .../nlp/sempre/paraphrase/Interval.java | 52 - .../paraphrase/NearestNeighborLearner.java | 344 ----- .../nlp/sempre/paraphrase/NnBuilder.java | 66 - .../sempre/paraphrase/ParaphraseBuilder.java | 34 - .../sempre/paraphrase/ParaphraseDataset.java | 254 ---- .../paraphrase/ParaphraseDerivation.java | 137 -- .../sempre/paraphrase/ParaphraseExample.java | 165 --- .../ParaphraseFeatureExtractor.java | 85 -- .../paraphrase/ParaphraseFeatureMatcher.java | 31 - .../sempre/paraphrase/ParaphraseLearner.java | 130 -- .../nlp/sempre/paraphrase/ParaphraseMain.java | 92 -- .../sempre/paraphrase/ParaphraseParser.java | 245 ---- .../sempre/paraphrase/ParaphraseUtils.java | 179 --- .../nlp/sempre/paraphrase/ParsingExample.java | 193 --- .../paraphrase/ParsingExampleProcessor.java | 120 -- .../nlp/sempre/paraphrase/Prediction.java | 40 - .../sempre/paraphrase/QuestionGenerator.java | 593 -------- .../sempre/paraphrase/VectorSpaceModel.java | 288 ---- .../paralex/ParalexQuestionReader.java | 47 - .../paraphrase/paralex/ParalexRules.java | 190 --- .../paraphrase/paralex/PhraseTable.java | 63 - .../nlp/sempre/paraphrase/rules/LangItem.java | 191 --- .../sempre/paraphrase/rules/LanguageExp.java | 112 -- .../paraphrase/rules/LanguageExpToken.java | 82 -- .../sempre/paraphrase/rules/LemmaPosRule.java | 57 - .../paraphrase/rules/LemmaPosSequence.java | 83 -- .../paraphrase/rules/ParaphraseAlignment.java | 73 - .../paraphrase/rules/RuleApplication.java | 53 - .../sempre/paraphrase/rules/RuleApplier.java | 435 ------ .../nlp/sempre/paraphrase/rules/Rulebase.java | 202 --- .../rules/SubstitutionRuleExtractor.java | 83 -- .../paraphrase/rules/SyntacticRuleSet.java | 322 ----- .../rules/VerbSemClassExtractor.java | 131 -- .../paraphrase/rules/VerbSemClassMatcher.java | 97 -- .../nlp/sempre/test/DerivationStreamTest.java | 32 + .../stanford/nlp/sempre/test/GrammarTest.java | 53 + .../nlp/sempre/test/GrammarValidityTest.java | 57 + .../nlp/sempre/test/JavaExecutorTest.java | 24 +- .../stanford/nlp/sempre/test/JsonTest.java | 63 +- .../stanford/nlp/sempre/test/LexiconTest.java | 87 -- .../stanford/nlp/sempre/test/ParserTest.java | 155 ++- .../stanford/nlp/sempre/test/SemTypeTest.java | 67 +- .../nlp/sempre/test/SemanticFnTest.java | 105 +- .../nlp/sempre/test/SystemSanityTest.java | 12 +- .../stanford/nlp/sempre/test/TestUtils.java | 40 +- .../nlp/sempre/test/TypeInferenceTest.java | 117 ++ .../stanford/nlp/sempre/vis/BeamFigures.java | 226 --- .../nlp/sempre/vis/ConfusionMatrices.java | 127 -- .../nlp/sempre/vis/ExampleDerivations.java | 260 ---- src/edu/stanford/nlp/sempre/vis/Utils.java | 119 -- testng.xml | 9 +- unittest-files/README | 1 + .../binaryInfoStringAndAlignment.txt | 5 + .../unaryInfoStringAndAlignment.txt | 4 + 308 files changed, 20414 insertions(+), 15628 deletions(-) delete mode 100644 .gitignore create mode 100644 DOCUMENTATION.md delete mode 100644 QUICKSTART.md create mode 100644 data/tutorial-arithmetic.examples rename data/{tutorial.grammar => tutorial-arithmetic.grammar} (60%) delete mode 100644 data/tutorial.examples.json delete mode 100644 data/unittest-learn.examples.json create mode 100644 demo-www/bullet.png create mode 100644 demo-www/index.html create mode 100644 demo-www/main.css create mode 100644 demo-www/main.js delete mode 100755 download-dependencies create mode 100644 freebase/README.md create mode 100644 freebase/data/emnlp2013.grammar create mode 100644 freebase/data/free917.test.examples.canonicalized.json create mode 100644 freebase/data/free917.train.examples.canonicalized.json create mode 100644 freebase/data/tutorial-freebase.grammar rename {data => freebase/data}/tutorial.ttl (93%) rename {data => freebase/data}/unittest-learn-ccg.grammar (100%) create mode 100644 freebase/data/unittest-learn.examples rename {data => freebase/data}/unittest-learn.grammar (100%) rename {scripts => freebase/scripts}/extract-freebase-schema.rb (93%) create mode 100755 freebase/scripts/fbshell.rb rename {scripts => freebase/scripts}/virtuoso (79%) create mode 100644 module-classes.txt delete mode 100755 parasempre create mode 100755 pull-dependencies delete mode 100644 release-acl2014.files delete mode 100644 release-code.files delete mode 100644 release-core.files delete mode 100644 release-emnlp2013.files delete mode 100644 release-fullfreebase.files delete mode 100644 release-fullfreebase_ttl.files delete mode 100644 release-fullfreebase_vdb.files delete mode 100644 release-geofreebase.files delete mode 100644 release-geofreebase_ttl.files delete mode 100644 release-geofreebase_vdb.files create mode 100755 run create mode 100755 scripts/agenda-stats create mode 100755 scripts/checkstyle.sh create mode 100644 scripts/checkstyle.xml create mode 100755 scripts/evaluation.py create mode 100755 scripts/extract-module-classes.rb create mode 100755 scripts/find-first-pred-diff.sh create mode 100755 scripts/find-hard-coded-paths.rb create mode 100755 scripts/fix-checkstyle.rb create mode 100755 scripts/tunnel create mode 100755 scripts/verify-code-loop.rb delete mode 100755 sempre create mode 100644 src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java create mode 100644 src/edu/stanford/nlp/sempre/ArithmeticFormula.java create mode 100644 src/edu/stanford/nlp/sempre/AtomicSemType.java create mode 100644 src/edu/stanford/nlp/sempre/BadFormulaException.java create mode 100644 src/edu/stanford/nlp/sempre/BoundedPriorityQueue.java delete mode 100644 src/edu/stanford/nlp/sempre/BridgeFn.java create mode 100644 src/edu/stanford/nlp/sempre/CallTypeInfo.java create mode 100644 src/edu/stanford/nlp/sempre/CanonicalNames.java create mode 100644 src/edu/stanford/nlp/sempre/ChartParserState.java create mode 100644 src/edu/stanford/nlp/sempre/CoarseParser.java create mode 100644 src/edu/stanford/nlp/sempre/Colorizer.java create mode 100644 src/edu/stanford/nlp/sempre/ContextFn.java create mode 100644 src/edu/stanford/nlp/sempre/ContextValue.java create mode 100644 src/edu/stanford/nlp/sempre/DerivOpCountFeatureComputer.java delete mode 100644 src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java delete mode 100644 src/edu/stanford/nlp/sempre/DerivationConstraint.java create mode 100644 src/edu/stanford/nlp/sempre/DerivationStream.java delete mode 100644 src/edu/stanford/nlp/sempre/Evaluation.java create mode 100644 src/edu/stanford/nlp/sempre/ExactValueEvaluator.java create mode 100644 src/edu/stanford/nlp/sempre/ExampleUtils.java create mode 100644 src/edu/stanford/nlp/sempre/FeatureComputer.java create mode 100644 src/edu/stanford/nlp/sempre/FloatingFeatureComputer.java create mode 100644 src/edu/stanford/nlp/sempre/FloatingParser.java create mode 100644 src/edu/stanford/nlp/sempre/FloatingRuleUtils.java delete mode 100644 src/edu/stanford/nlp/sempre/FormulaGenerationInfo.java delete mode 100644 src/edu/stanford/nlp/sempre/FormulaRetriever.java delete mode 100644 src/edu/stanford/nlp/sempre/FreebaseInfo.java create mode 100644 src/edu/stanford/nlp/sempre/FuncSemType.java create mode 100644 src/edu/stanford/nlp/sempre/FuzzyMatchFn.java create mode 100644 src/edu/stanford/nlp/sempre/HasScore.java create mode 100644 src/edu/stanford/nlp/sempre/KnowledgeGraph.java create mode 100644 src/edu/stanford/nlp/sempre/LanguageAnalyzer.java delete mode 100644 src/edu/stanford/nlp/sempre/LexiconFn.java create mode 100644 src/edu/stanford/nlp/sempre/MultipleDerivationStream.java create mode 100644 src/edu/stanford/nlp/sempre/NaiveKnowledgeGraph.java create mode 100644 src/edu/stanford/nlp/sempre/NullTypeLookup.java create mode 100644 src/edu/stanford/nlp/sempre/ParaphraseModel.java create mode 100644 src/edu/stanford/nlp/sempre/ParserAgenda.java delete mode 100644 src/edu/stanford/nlp/sempre/Parsers.java create mode 100644 src/edu/stanford/nlp/sempre/ReinforcementParser.java create mode 100644 src/edu/stanford/nlp/sempre/ReinforcementUtils.java create mode 100644 src/edu/stanford/nlp/sempre/SemTypeHierarchy.java create mode 100644 src/edu/stanford/nlp/sempre/SempreUtils.java create mode 100644 src/edu/stanford/nlp/sempre/Server.java create mode 100644 src/edu/stanford/nlp/sempre/SimpleAnalyzer.java create mode 100644 src/edu/stanford/nlp/sempre/SimpleLexicon.java create mode 100644 src/edu/stanford/nlp/sempre/SimpleLexiconFn.java create mode 100644 src/edu/stanford/nlp/sempre/SingleDerivationStream.java delete mode 100644 src/edu/stanford/nlp/sempre/StringCache.java create mode 100644 src/edu/stanford/nlp/sempre/TableValue.java create mode 100644 src/edu/stanford/nlp/sempre/TopSemType.java create mode 100644 src/edu/stanford/nlp/sempre/TypeInference.java create mode 100644 src/edu/stanford/nlp/sempre/TypeLookup.java create mode 100644 src/edu/stanford/nlp/sempre/UnionSemType.java create mode 100644 src/edu/stanford/nlp/sempre/ValueEvaluator.java delete mode 100644 src/edu/stanford/nlp/sempre/Vis.java create mode 100644 src/edu/stanford/nlp/sempre/build.xml create mode 100644 src/edu/stanford/nlp/sempre/cache/FileStringCache.java create mode 100644 src/edu/stanford/nlp/sempre/cache/LruCallback.java create mode 100644 src/edu/stanford/nlp/sempre/cache/LruMap.java create mode 100644 src/edu/stanford/nlp/sempre/cache/RemoteStringCache.java create mode 100644 src/edu/stanford/nlp/sempre/cache/StringCache.java rename src/edu/stanford/nlp/sempre/{ => cache}/StringCacheServer.java (83%) rename src/edu/stanford/nlp/sempre/{ => cache}/StringCacheUtils.java (64%) create mode 100644 src/edu/stanford/nlp/sempre/cache/build.xml create mode 100644 src/edu/stanford/nlp/sempre/cache/test/StringCacheTest.java create mode 100644 src/edu/stanford/nlp/sempre/corenlp/CoreNLPAnalyzer.java create mode 100644 src/edu/stanford/nlp/sempre/corenlp/build.xml create mode 100644 src/edu/stanford/nlp/sempre/corenlp/test/CoreNLPSemanticFnTest.java delete mode 100644 src/edu/stanford/nlp/sempre/fbalignment/lexicons/BinaryLexicon.java delete mode 100644 src/edu/stanford/nlp/sempre/fbalignment/lexicons/UnaryLexicon.java delete mode 100644 src/edu/stanford/nlp/sempre/fbalignment/lexicons/WordDistance.java delete mode 100644 src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/EntryNormalizer.java delete mode 100644 src/edu/stanford/nlp/sempre/fbalignment/utils/CollectionUtils.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/BinaryLexicon.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/BridgeFn.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/BuildCanonicalIdMap.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/BuildTypesMap.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/CanonicalizeExamples.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/CanonicalizeIds.java rename src/edu/stanford/nlp/sempre/{fbalignment/lexicons => freebase}/EntityLexicon.java (63%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/ExecuteExamples.java rename src/edu/stanford/nlp/sempre/{ => freebase}/FbFormulasInfo.java (60%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/FilterFreebase.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/Free917Converter.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/FreebaseInfo.java rename src/edu/stanford/nlp/sempre/{ => freebase}/FreebaseSearch.java (79%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/FreebaseTypeLookup.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/FreebaseValueEvaluator.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/LambdaCalculusConverter.java rename src/edu/stanford/nlp/sempre/{fbalignment/lexicons => freebase}/Lexicon.java (67%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/LexiconFn.java rename src/edu/stanford/nlp/sempre/{ => freebase}/SparqlExecutor.java (51%) rename src/edu/stanford/nlp/sempre/{ => freebase}/SparqlExpr.java (77%) rename src/edu/stanford/nlp/sempre/{ => freebase}/Stemmer.java (87%) rename src/edu/stanford/nlp/sempre/{ => freebase}/TextToTextMatcher.java (67%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/UnaryLexicon.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/build.xml rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/index/FbEntityIndexer.java (96%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/index/FbEntitySearcher.java (93%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/index/FbIndexField.java (83%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/EntrySource.java (85%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/ExtremeValueWrapper.java (92%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/LexicalEntry.java (81%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/TokenLevelMatchFeatures.java (96%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/normalizers/BinaryNormalizer.java (97%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/EntryNormalizer.java rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/normalizers/IdentityNormalizer.java (67%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/lexicons/normalizers/PrepDropNormalizer.java (92%) rename src/edu/stanford/nlp/sempre/{ => freebase}/test/FbFormulasTest.java (66%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/test/FreebaseInfoTest.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/test/FreebaseSemTypeTest.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/test/FreebaseTypeInferenceTest.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/test/LexiconTest.java rename src/edu/stanford/nlp/sempre/{ => freebase}/test/PrepDropNormalizerTest.java (52%) rename src/edu/stanford/nlp/sempre/{ => freebase}/test/SparqlExecutorTest.java (78%) rename src/edu/stanford/nlp/sempre/{ => freebase}/test/StemmerTest.java (78%) rename src/edu/stanford/nlp/sempre/{ => freebase}/test/TokenMatchTest.java (83%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/utils/CollectionUtils.java rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/utils/DoubleContainer.java (93%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/utils/FileUtils.java (93%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/utils/FormatConverter.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/utils/FreebaseUtils.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/utils/LinkedExtractionFileUtils.java rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/utils/MathUtils.java (74%) create mode 100644 src/edu/stanford/nlp/sempre/freebase/utils/SemparseLogTools.java create mode 100644 src/edu/stanford/nlp/sempre/freebase/utils/ShortContainer.java rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/utils/WnExpander.java (93%) rename src/edu/stanford/nlp/sempre/{fbalignment => freebase}/utils/WordNet.java (85%) delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/Aligner.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/Context.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ContextModel.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ContextSimilarityModel.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/DatasetUtils.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/EntityInstance.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/EntityModel.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarity.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarityComputer.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/Interval.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/NearestNeighborLearner.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/NnBuilder.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseBuilder.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDataset.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDerivation.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseExample.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureExtractor.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureMatcher.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseLearner.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseMain.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseParser.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParaphraseUtils.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParsingExample.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/ParsingExampleProcessor.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/Prediction.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/QuestionGenerator.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/VectorSpaceModel.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexQuestionReader.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexRules.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/paralex/PhraseTable.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/LangItem.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExp.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExpToken.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosRule.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosSequence.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/ParaphraseAlignment.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplication.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplier.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/Rulebase.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/SubstitutionRuleExtractor.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/SyntacticRuleSet.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassExtractor.java delete mode 100644 src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassMatcher.java create mode 100644 src/edu/stanford/nlp/sempre/test/DerivationStreamTest.java create mode 100644 src/edu/stanford/nlp/sempre/test/GrammarTest.java create mode 100644 src/edu/stanford/nlp/sempre/test/GrammarValidityTest.java delete mode 100644 src/edu/stanford/nlp/sempre/test/LexiconTest.java create mode 100644 src/edu/stanford/nlp/sempre/test/TypeInferenceTest.java delete mode 100644 src/edu/stanford/nlp/sempre/vis/BeamFigures.java delete mode 100644 src/edu/stanford/nlp/sempre/vis/ConfusionMatrices.java delete mode 100644 src/edu/stanford/nlp/sempre/vis/ExampleDerivations.java delete mode 100644 src/edu/stanford/nlp/sempre/vis/Utils.java create mode 100644 unittest-files/README create mode 100644 unittest-files/binaryInfoStringAndAlignment.txt create mode 100644 unittest-files/unaryInfoStringAndAlignment.txt diff --git a/.gitignore b/.gitignore deleted file mode 100644 index a51a034..0000000 --- a/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -lib -state -test-output -*.cache - -virtuoso-opensource - -data/free917*.json -data/emnlp*.grammar diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..56f2e29 --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,1237 @@ +# SEMPRE documentation + +This document describes SEMPRE in detail. See +[TUTORIAL.md](https://github.com/percyliang/sempre/blob/master/TUTORIAL.md) for +a more engaging introduction. This document assumes a modest understanding of +how SEMPRE works. + +If you find any bugs, please report them or fix them (file a GitHub issue or +submit a pull request)! + +## Code organization + +SEMPRE is designed to be modular so that one can plug in various types of +parsers, logical forms, executors, etc. + +### Modules + +The SEMPRE code is broken up into a set of modules (see `Makefile` to see the +list). The code of the core module is in: + + src/edu/stanford/nlp/sempre + +The code for each of the other non-essential modules is in: + + src/edu/stanford/nlp/sempre/ + +- core: contains all the basic code (this is kept to a minimum) +- cache: allows us to run a cache server (implements a simple key-value store for things like SPARQL results) +- corenlp: contains the glue code that allows us to use Stanford CoreNLP +- freebase: needed to handle logical forms which are database queries (SPARQL, Freebase) + +### Java classes + +Here are the basic Java classes in `core`, organized by function: + +Logical forms: + +- `Formula` (`ValueFormula`, `SuperlativeFormula`, ...): logical form +- `Derivation`: contains `Formula` and the way it was constructed +- `SemanticFn` (`JoinFn`, `NumberFn`, ...): takes multiple `Derivation`s and combines them into one +- `Rule`: associated with a `SemanticFn` which specifies how to combine `Derivation`s +- `Grammar`: a set of `Rule`s + +Execution: + +- `Value` (`NameValue`, `ListValue`, ...): represents a denotation +- `Executor` (`JavaExecutor`, `freebase.SparqlExecutor`): takes a `Formula` and returns a value +- `ValueEvaluator` (`ExactValueEvaluator`, `freebase.FreebaseValueEvaluator`): evaluates how correct a denotation is (with respect to the correct answer) + +Parsing: + +- `Example`: specifies an utterance and a `ContextValue` +- `LanguageAnalyzer` (`SimpleLanguageAnalyzer`, `corenlp.CoreNLPAnalyzer`): does basic linguistic analysis on an `Example` +- `FeatureExtractor`: takes an `Example` and `Derivation` and extracts features for scoring +- `Params`: parameters of the semantic parser +- `Parser` (`BeamParser`, `FloatingParser`, `ReinforcementParser`): takes `Example`s and produces `Derivation`s, scoring them using `FeatureExtractor` and `Params` + +Learning: + +- `Dataset`: specifies a set of `Example`s +- `Learner`: takes a `Dataset` and produces parameters `Params` + +# Logical forms + +There are two types of logical forms, depending on the application: + +- Lambda DCS logical forms for querying databases (declarative); use `freebase.SparqlExecutor` +- Simple Java logical forms for doing everything else (procedural); use `JavaExecutor` + +The interpretation/denotation of a logical form depends on the `Executor` that +is being used (specified via the `-Builder.executor` option). `Executor`s +(i.e., classes that extend `Executor`) define how logical forms are mapped to +denotations. `JavaExecutor` treats the logical forms as simple Java +code and just runs it. `SparqlExecutor` treats the logical forms as lambda DCS +expressions, which are used to query a database (e.g., via SPARQL, and e.g., +Freebase). + +Logical forms are defined recursively. In the base case, we have primitive +formulas, which are either variables are values. In the recursive case, we +build up larger formulas out of smaller formulas using various composition +constructs. It is useful to think of the logical form as a tree, where the +leaves are the primitive formulas and each non-leaf subtree as corresponpding +to one composition operation. + +## Primitive logical forms + +### ValueFormula + +A `ValueFormula` is a formula that wraps a specific denotation `Value`. +The possible `Value`s, along with an example are as follows: + +- `BooleanValue`: standard boolean value. For example: + + (boolean true) + (boolean false) + +- `NumberValue`: standard floating point number with optional units. For example: + + (number 2) + (number 2.5) + (number 2.5 fb:en.meter) + +- `StringValue`: standard UTF-8 string. For example: + + (string hello) + (string "hello") + (string "hello world") + (string hello\ world) + + Note that the first two and the last two are identical. + +- `DateValue`: year-month-day representation of a date. For example: + + (date 2014 12 21) # December 21, 2014 + (date 2014 -1 -1) # 2014 + (date -1 12 21) # December 21 + (date -1 12 -1) # December + + In general: + + (date ) + +- `NameValue`: represents a predicate symbol (either representing an entity or relation in a knowledge base); + the symbol has an optional description. For example: + + fb:en.barack_obama + (name fb:en.barack_obama) + (name fb:en.barack_obama "Barack Obama") + + all refer to the same symbol. Note: the first one is only valid as a string + representation of a `Formula` (which represent a `Value`) — to be used + in logical forms. The second two are valid representations of a `Value` + — usually returned from `Executor`s. + +- `ListValue`: represents a list of values. For example: + + (list) + (list (number 1) (number 2) (string hello)) + + In general: + + (list ... ) + +- `TableValue`: represents a table (matrix) of values, where each column has a string. Here is an example table: + + (table (State Capital) ((name fb:en.california) (name fb:en.sacramento)) ((name fb:en.oregon) (name fb:en.salem))) + + In general: + + (table ( ... ) ( ... ) ...) + +There are some more arcane `Value`s (see `Values.java` for a list), but they +are not that important from the point of view of specifying a logical form. + +### VariableFormula + +Besides `ValueFormula`s, the other primitive formula is a `VariableFormula`, +which represent variables in lambda expressions. An example `VariableFormula` is: + + (var x) + +which simply represents a variable with the name `x`. On their own, these +formulas are rather uninteresting. However, they are integral subcomponents of +`LambdaFormula`s which are described later. + +## Compositional logical forms + +Now we describe the compositional logical forms, those that allow us to +construct larger logical forms from smaller ones. + +For executing simple Java programs (`JavaExecutor`), the only relevant +composition is `CallFormula`. The rest are for building lambda DCS documents +for execution using the `freebase.SparqlExecutor`. + +### CallFormula + +A `CallFormula` has the following form: + + (call ... ) + +The `` is a string specifying any Java function and the +`` entries are logical forms. The function is one of the following: + +- A static method: + + (call java.lang.Math.cos (number 0)) + +- An instance method (`arg-1` is used as `this`): + + (call .length (string hello)) # (number 5) + (call .indexOf (string "what is this?") (string is)) # (number 5) + +- A shortcut, which maps onto to a static method in `JavaExecutor.BasicFunctions`: + + (call < (number 3) (number 4)) # (boolean true) + (call + (number 3) (number 4)) # (number 7) + (call + (string 3) (string 4) (string 5)) # (string 345) + + Note that operator overloading is supported, and resolution is based on the + argument types. Conditionals are supported but both true and false branches + are evaluated: + + (call if (call < (number 3) (number 4)) (string yes) (string no)) # (string yes) + + We can perform iteration via functional programming. These list functions + take a `ListValue` and a `LambdaFormula` as arguments: + + (call map (list (number 3) (number 4)) (lambda x (call * (var x) (var x)))) # (list (number 9) (number 16)) + (call select (list (number 3) (number 4)) (lambda x (call < (var x) (number 3.5))))) # (list (number 3)) + (call reduce (list (number 3) (number 4)) (lambda x (lambda y (call + (var x) (var y)))))) # (number 7) + + To create a list of indices: + + (call range (number 0) (number 3)) # (list (number 0) (number 1) (number 2)) + + Note that the goal is not to support the ability to write arbitrarily complex + programs using this fairly verbose language. In practice, you would write + your own custom modules, and use these mechanisms to glue a small set of modules together. + +### JoinFormula + +Now we embark on our tour of lambda DCS logical forms. See the [lambda DCS +documentation](http://arxiv.org/pdf/1309.4408.pdf) for a more mathematical +description. + +A brief note about terminology in lambda DCS. Unaries and binaries are logical +forms which represent sets and (binary) relations: + +- Unary (e.g., `(fb:type.object.type fb:people.person)`): denotes a set of entities +- Binary (e.g., `fb:location.location.area`): denotes a set of entity-pairs (includes functions too). + We consider the first argument the *head* and the second argument the *modifier*. For example: + + fb:location.location.containedby denotes {(fb:en.san_francisco, fb:en.california), ...} + fb:people.person.place_of_birth denotes {(fb:en.barack_obama, fb:en.honolulu), ...} + !fb:people.person.place_of_birth denotes {(fb:en.honolulu, fb:en.barack_obama), ...} + + The notational convention is that `!` reverses the head and modifier. + + In general: + + denotes {(, ), ...} + +A `JoinFormula` combines a binary with a unary: + + ( ) + +This is a database join on the second argument of the binary and the unary, and +a projection onto the first argument of the binary. + +Here are some examples: + + (fb:location.location.containedby fb:en.california) + (!fb:people.person.place_of_birth fb:en.barack_obama) + (fb:people.person.place_of_birth (fb:location.location.containedby fb:en.california)) + +Sometimes, the binary is special, and the resulting logical form denotes an +infinite set: + + (!= fb:en.california) # denotes entities not equal to California + (> (number 5)) # denotes numbers greater than 5 + (STRSTARTS (string A)) # denotes strings starting with "A" + +Note: One might be tempted to think of `JoinFormula` as function application. +This is sometimes valid, but one has to be very careful about which is the head +and which is the modifier. When the binary is a `LambdaFormula`, then function +application is generally the right mental model. + +### MergeFormula + +A `MergeFormula` combines two formulas and represents either their set +intersection (and) or set union (or): + + (and ) + (or ) + +For example, *scientists born in Seattle*: + + (and (fb:people.person.profession fb:en.scientist) (fb:people.person.place_of_birth fb:en.seattle)) + +Here are *people who are born in Honolulu or Seattle*: + + (or (fb:people.person.place_of_birth fb:en.honolulu) (fb:people.person.place_of_birth fb:en.seattle)) + +### NotFormula + +A `NotFormula` takes a formula denoting a set and denotes the complement of that set: + + (not ) + +For example, *cities not in California*: + + (and (fb:type.object.type fb:location.citytown) (not (fb:location.location.containedby fb:en.california))) + +### AggregateFormula + +An `AggregateFormula` takes a formula representing a set and performs some operation. The general form is: + + ( ) + +where `` defines the type of aggregation to be performed while +`` denotes/defines the set that we are aggregating over. The possible +modes (with their corresponding semantics) are listed below: + +- `count`: Returns the cardinality of a set. +- `sum` : Returns the sum of the elements in the set. +- `avg` : Returns the average/mean of the elements in the set. +- `min` : Returns the minimum element of a set. +- `max` : Returns the maximum element of a set. + +Note that the latter four modes can only be applied to logical forms denoting sets of numbers. + +For example, *number of people born in Honolulu*: + + (count (fb:people.person.place_of_birth fb:en.honolulu)) # (number 570) + +Here is the *maximum height of any mountain* + + (max (!fb:geography.mountain.elevation (fb:type.object.type fb:geography.mountain))) # (number 8848 fb:en.meter) + +### ArithmeticFormula + +An `ArithmeticFormula` combines two logical forms denoting numbers and also +denotes a number. + +The form of an `ArithmeticFormula` is as follows: + + ( ) + +The `` entry is either `+`, `-`, `*`, or `/`, and the two arguments +are formulas that denote numeric values (including dates, although the support +there is a bit sketchy). Here are some examples: + + (+ (number 5) (number 3)) # (number 8) + (* (number -1) (number 3)) # (number -3) + +Here's how to compute the difference in height between two people: + + (- (!fb:people.person.height_meters fb:en.michael_jordan) (!fb:people.person.height_meters fb:en.barack_obama)) # (number 0.130 fb:en.meter) + +Note that we could not define these arithmetic formulas via `JoinFormula` +because they are ternary relations rather than binary ones. + +### ReverseFormula + +A `ReversalFormula` takes a binary logical form denoting a set of head-modifier +pairs and denotes the corresponding set of modifier-head pairs. + +Recall: + + fb:people.person.place_of_birth denotes {(fb:en.barack_obama, fb:en.honolulu), ...} + +The following is equivalent to `!fb:people.person.place_of_birth`: + + (reverse fb:people.person.place_of_birth) denotes {(fb:en.honolulu, fb:en.barack_obama), ...} + +But reversal can be applied to compositional binaries built using `LambdaFormula`: + + (reverse (lambda x (fb:people.person.places_lived (fb:people.place_lived.location (var x))))) + +You can check that the above is equivalent to: + + (lambda x (!fb:people.place_lived.location (!fb:people.person.places_lived (var x)))) + +In both logical forms, the location is in the head position and person is in +the modifier position. + +### LambdaFormula + +Now the serious stuff begins. Up until now, all the compositional logical +forms were unaries (which denote sets). Lambda abstraction allows us to +construct logical forms that denote binary, ternary, and +other higher-order relations. + +The general form includes a variable and body formula which uses the variable: + + (lambda ) + +If the body is a unary, then the resulting logical form is a binary, where the +unary represents the head and the variable represents the modifier. Let us see some examples. + +The first one is equivalent to `fb:people.person.place_of_birth`: + + (lambda x (fb:people.person.place_of_birth (var x))) + +This logical form denotes a binary relation where the head is the person and +the modifier is the location: + + (lambda x (fb:people.person.places_lived (fb:people.place_lived.location (var x)))) + +The more complex (but very useful) example represents a binary where the head +is a person and the modifier is the number of children the person has: + + (reverse (lambda x (count (!fb:people.person.children (var x))))) denotes {(fb:en.barack_obama, 2), ...} + +#### Macro substitution + +Another more syntactic view of `LambdaFormula`s is that they are just logical +forms with holes. In this case, a `JoinFormula` where the binary is a +`LambdaFormula` performs macro substitution. Here is an example of a join +of a `LambdaFormula` binary and an unary: + + ((lambda x (fb:people.person.places_lived (fb:people.place_lived.location (var x)))) fb:en.seattle) + +In the macro substitution view, the variable `x` becomes bound to `fb:en.seattle`, and the resulting logical form is: + + (fb:people.person.places_lived (fb:people.place_lived.location fb:en.seattle)) + +which is equivalent. This process is called beta-reduction. But one must +exercise care, since the macro substitution view and the original higher-order +relation view are different. Consider: + + ((lambda x (and (!fb:people.person.place_of_birth (var x)) (!fb:people.deceased_person.place_of_death (var x)))) (fb:type.object.type fb:people.person)) + +In the higher-order relation view, the binary relates locations (head) to +people (modifier). The resulting logical form denotes the set of locations +which are both the place of birth and place of death of some person. However, +if we do macro substitution, then we get the set of locations which are either +the place of birth of someone or the place of death of someone (possibly +different). + +Macro substitution is triggered by explicit beta reduction (see `JoinFn` with +`betaReduce` below), and this is typically during the intermediate steps of +constructing logical forms. + +We can construct logical forms that have more than one argument: + + (lambda binary (lambda unary ((var binary) (var unary)))) + +We can apply this ternary logical form two two arguments, + + (((lambda binary (lambda unary ((var binary) (var unary)))) fb:people.person.place_of_birth) fb:en.seattle) + +which after beta reduction results in the very mundane + + (fb:people.person.place_of_birth fb:en.seattle) + +In lambda DCS, all logical forms are either unaries or binaries. Ternaries are +beyond are exclusively used for macro substitution to construct logical forms +in a controlled compositional way. + +### SuperlativeFormula + +A `SuperlativeFormula` formula takes a unary denoting a set and a binary +denoting a relation between elements of that set (head) and a number +(modifier), and denotes a set representing extreme elements based on the +relation. + +Here is the general form: + + ( ) + +The different pieces are as follows: + +- `` is either `argmin` or `argmax` +- `` is an integer indicating the offset in the sorted list of the entities +- `` specifies the number of elements to return +- `` is the base set that we're drawing from +- `` specifies the relation by which we sort the elements + +Here is *the city with the largest area*: + + (argmax 1 1 (fb:type.object.type fb:location.citytown) fb:location.location.area) + +The *second largest city by area*: + + (argmax 2 1 (fb:type.object.type fb:location.citytown) fb:location.location.area) + +The *five largest cities by area*: + + (argmax 1 5 (fb:type.object.type fb:location.citytown) fb:location.location.area) + +The *person who has the most number of children* (who, in Freebase, turns out +to be Chulalongkorn with a whopping 62): + + (argmax 1 1 (fb:type.object.type fb:people.person) (reverse (lambda x (count (!fb:people.person.children (var x)))))) + +### MarkFormula + +In a way, `MarkFormula` essentially allows us to do anaphora. If we think of +simple lambda DCS (joins and merges) as producing tree-structured logical +forms, `MarkFormula` allows us to have non-tree edges between the root of a +subtree and a node in that subtree. + +Recall `LambdaFormula` creates a binary from a unary. A `MarkFormula` creates +a unary from a unary, which simply binds the body unary to the variable (which +appears in the body). + +The general form: + + (mark ) + +Here is *those whose place of birth is the same as their place of death*: + + (mark x (fb:people.person.place_of_birth (!fb:people.deceased_person.place_of_death (var x)))) + +Note that `x` binds to the head of `fb:people.person.place_of_birth`, +representing the person in question. Using allows us to use this head again +deeper in the logical form. Here's a pictorial representation: + + x -- [fb:people.person.place_of_birth] --> ? + ? -- [!fb:people.deceased_person.place_of_death] --> x + +where `?` denotes an unnamed intermediate variable. + +# Semantic functions + +From the tutorial, recall that a grammar is a set of rules of the following +form: + + (rule ( ... ) ) + +Basically, these rules specify how derivations belonging to the source categories are combined +into derivations that correspond to the target category. +Semantic functions are the workhorses of these rules: +they take in a set of source derivations and apply transformations (defined via Java code) on these derivations +in order to produce a derivation that is a member of the target category. + +Since semantic functions are built from arbitrary Java code (any class extending `SemanticFn`), there is a great deal of flexibility, +and knowing how to properly use semantic functions is perhaps the most important step in working with SEMPRE. +This section defines the semantic functions that come pre-packaged with SEMPRE. +In all likelihood, these semantic functions will provide all the functionality that you need. + +Some of these semantic functions will rely on other packages/classes/options being used/set in order to function properly. +For example, many semantic functions require that the Stanford CoreNLP package is loaded (via the `-languageAnalyzer corenlp.CoreNLPAnalyzer` option); +this package contains many general purpose NLP algorithms that extract linguistic information (e.g., parts of speech) from utterances. +Dependences on packages such as CoreNLP will be noted when necessary. + +## Special categories: + +Some special categories that you should now in order to effectively write grammars and use semantic functions: + +- `$TOKEN`: selects a single atomic word from the utterance. +- `$PHRASE`: selects any subsequence of tokens/words from the utterance. +- `$LEMMA_TOKEN`: selects any atomic word from the utterance and lemmatizes the word. For example, "arriving" would become "arrive". +- `$LEMMA_PHRASE`: selects any subsequence of tokens/words and lemmatizes them. + +Both of the special categories that rely on lemmatization will only function +properly if the `corenlp.CoreNLPAnalyzer` is loaded. Otherwise, the +lemmatization will simply amount to making everything lowercase. + +There are two broad informal classes of semantic functions: primitive semantic +functions and compositional semantic functions. The primitive ones take +phrases in the natural language utterance, filters them, and produces some +derivation generally with a simple logical form. The compositional ones take +these simple derivations and combines them into larger derivations, +usually generating larger logical forms. + +## Primitive semantic functions + +Primitives are basic semantic functions that directly map spans (i.e., subsequences) of an utterance to logical forms. +Most grammars will rely on primitives as the "bottom" rules, i.e., the rules that have some span of the utterance as their right hand side (RHS) +and which will other rules build off of. + +### ConstantFn + +This is a basic primitive function which allows for you to hard-code basic rules, such as + + (rule $ROOT (barack obama) (ConstantFn fb:en.barack_obama fb:people.person)) + +which would parse the following utterance into its corresponding entity (i.e., logical form): + + barack obama # (name fb:en.barack_obama) + +Note the form that `ConstantFn` takes, i.e. `(ConstantFn formula type)`, and that the `type` argument is optional, meaning that + + (rule $ROOT (barack obama) (ConstantFn fb:en.barack_obama)) + +would also parse the above utterance. +However, note that if types are not explicitly added, the system relies on automatic type inference (see the section on Types below). + +Another example of `ConstantFn` would be the following: + + (rule $ROOT (born in) (ConstantFn fb:people.person.place_of_birth (-> fb:location.location fb:people.person)))) + +which would parse the phrase `born in` to a relation/binary logical form (as opposed to an entity) as follows: + + born in # (name fb:people.person.place_of_birth) + +Lastly, note that in both these cases the parsed logical form is prefixed with `name`. +This denotes that the logical form has a `NameValue` denotation, which simply means that it represents a logical predicate/entity +and not some other primitive (e.g., a `NumberValue` or `DateValue`). + +### SimpleLexiconFn (reliant on having a SimpleLexicon) + +This function allows you to map phrases/tokens to logical entities in a more scalable manner than hard-coding everything with `ConstantFn` directly in your grammar. +Suppose you have a `SimpleLexicon` loaded from a JSON file containing the entries: + + {'lexeme' : 'barack obama', 'formula' : 'fb:en.barack_obama', 'type' : 'fb:people.person'} + {'lexeme' : 'born in', 'formula' : 'fb:people.person.place_of_birth', 'type' : '(-> fb:location.location fb:people.person)'} + +then the rule + + (rule $ROOT ($PHRASE) (SimpleLexiconFn (type fb:people.person))) + +will parse the following: + + barack obama # (name fb:en.barack_obama) + +and the rule + + (rule $ROOT ($PHRASE) (SimpleLexiconFn (type (-> fb:location.location fb:people.person)))) + +will parse + + born in # (name fb:people.person.place_of_birth) + +Thus, the function of `SimpleLexiconFn` is similar to `ConstantFn`, but it facilitates more modularity +since the lexical items are contained within the `SimpleLexicon` (loaded from a JSON) instead of being hard-coded into the grammar. + +In the above examples, this type annotation is not particularly important, but consider the case where we have the following lexical entries: + + {"lexeme" : "Lincoln", "formula" : "fb:en.abraham_lincoln", "type" : "fb:people.person"} + {"lexeme" : "Lincoln", "formula" : "fb:en.lincoln_nevada", "type" : "fb:location.location"} + +Both of these lexical entries have the same trigger phrase (i.e., lexeme). +However, they correspond to very distinct entities (one is a former president, while the other is a city in Nevada). +By leveraging the type annotation, we can specify which entity we actually want to trigger. +For example, the rule + + (rule $ROOT ($PHRASE) (SimpleLexiconFn (type fb:location.location))) + +would ensure that we only trigger the entity that corresponds to the city. +Alternatively, if we wanted both entities to be triggered (e.g., if type-checking later on will handle the ambiguity), +then we could write: + + (rule $ROOT ($PHRASE) (SimpleLexiconFn (type fb:type.any))) + +### NumberFn (partially reliant on the CoreNLPAnalyzer) + +`NumberFn` is the basic primitive function for parsing numbers from an utterance. +For example, + + (rule $ROOT (NumberFn)) + +would parse the following strings into the following logical forms: + + + 2.3 # (number 2) + four # (number 4) + 3 million # (number 3000000) + +The last one works only if `-languageAnalyzer corenlp.CoreNLPAnalyzer` is set. + +### DateFn (reliant on the CoreNLPAnalyzer) + +`DateFn` is the basic primitive function for parsing date values. +For example, + + (rule $ROOT (DateFn)) + +would parse the following strings into the following logical forms: + + Thursday, December 4th # (date -1 12 4) + The 12th of Nov, 2012 # (date 2012 11 12) + August # (date -1 8 -1) + +Note that the logical representation of dates here (defined in the `DateValue` class) is distinct from Java's date (or JodaTime etc.) +and that only dates not times are represented. +The logical form representation of a `DateValue` is simply `(date year month day)`, with one-based indexing for the months. +The above examples also illustrate how missing aspects of dates are treated: +if any part of a date (day, month, or year) is left unspecified, then the `DateValue` logical representation inserts -1s in those positions. + +## Filtering semantic functions + +You might find that your grammar is generating far too many candidate +derivations, resulting in the correct derivations falling off the beam. After +all, the number of derivations does tend to grow exponentially with the +sentence length. + +The semantic functions described below help to ameliorate this issue by +allowing you to filter down the phrases that are considered. In other words, +they help control how many logical forms you generate by allowing you to +specify more precise situations in which rules should fire. + +This is particularly important when your lexicon (like our EMNLP 2013 system) +contains a lot of noisy entries. + +### FilterSpanLengthFn + +This semantic function allows you to filter out the length of spans that you trigger on. For example, + + (rule $Length2Span ($PHRASE) (FilterSpanLengthFn 2)) + +produces a new category `$Length2Span` which contains only phrases of length 2, +which is useful for limiting the number of phrases you compute on. +For instance, we could combine this with the simple lexicon function we used above and say + + (rule $ROOT ($Length2Span) (SimpleLexiconFn (type fb:type.any))) + +and this would parse + + barack obama # (name fb:en.barack_obama) + +but would make running the grammar faster than if we replaced `$Length2Span` with `$PHRASE`, since `$PHRASE` will try all possible lengths, +and we know that our lexicon only contains length 2 phrases. + +### FilterPosTagFn (relies on CoreNLPAnalyzer) + +`FilterPosTagFn` allows you to extract spans or tokens that are composed only of certain parts of speech (POS) tags. +For example, + + (rule $ProperNoun ($TOKEN) (FilterSpanPosTag token NNP NNPS)) + +produces a new category `$ProperNoun` which contains words that are proper nouns. +This would accept phrases like: + + honolulu + obama + +and assign them to the `$ProperNoun` category. + +Note the options that are passed: `token` specifies that we want to look at single tokens and `NNP NNPS` are the POS tags for proper nouns. +If you wanted to get multi-word proper nouns then you would use + + (rule $ProperNounPhrase ($PHRASE) (FilterSpanPosTag span NNP NNPS)) + +Note that we both changed the RHS category to `$PHRASE` and changed the `token` option to `span`. +This would accept things like + + honolulu hawaii + barack obama + +but note that the entire span must have the same POS tag (that is in one of the accepted categories). +Also, it will only accept the maximal span, meaning that `barack obama` will NOT be parsed three times as `barack`, `obama`, and `barack obama`. + +Note also that, unlike the previous examples for simpler semantic functions, the logical form is not written out next to the phrases. +This is because the above rule to does not rewrite to `$ROOT` (i.e., it does not have `$ROOT` as a LHS), +and thus, this rule would accept the example phrases above, but it does not parse them directly to a logical form. +Other rules which take `$ProperNounPhrase` as a RHS are necessary to complete a grammar with this rule. +Most of the filtering functions discussed below will also have this flavor. + +### FilterNerSpanFn (relies on CoreNLPAnalyzer) + +This `SemanticFn` allows you to extract spans that are named entities (NEs). +For example, + + (rule $Person ($Phrase) (FilterNerSpan PERSON)) + +produces a new category (`$Person`) which contains phrases that the CoreNLPAnalyzer labels as referring to people. +This would accept phrases like + + barack obama + michelle obama + +Note that unlike our earlier examples where we parsed these phrases corresponding to an entity/person, +the LHS side of our rule in this case is a new category `$Person`. +The idea is that you would use this new category to restrict what you apply rules on further up in the derivation. +For example, you may want to change our earlier `SimpleLexiconFn` example rule to + + (rule $ROOT ($Person) (SimpleLexiconFn (type fb:people.person))) + +since you know that this rule should be restricted to spans that may contain a mention of a person. + +### ConcatFn + +This function allows you to concatenate strings, giving you more fine-grained control over how categories are constructed over spans, +which is useful for doing things that are not possible with the other filter functions. +Concretely, say you wanted to look for spans of the utterance that may correspond to binaries, +you would probably want to look at verbs (e.g., "lived", "born"). +However, triggering only on verbs is too restrictive; for example, some binaries take their meaning from verb-preposition pairs. +In fact, the example of "born in" used previously is exactly such a phrase. +In order to handle this case, we could use `ConcatFn` with the following rules: + + (rule $Verb ($TOKEN) (FilterSpanPosTag token VB VBD VBN VBG VBP VBZ)) + (rule $Prep ($TOKEN) (FilterSpanPosTag token IN)) + (rule $VerbPrep ($Verb $Prep) (ConcatFn " ")) + +These rules would create a new category (`$VerbPrep`) that contains two word phrases consisting of a verb followed by a preposition. +One way we could leverage this category would be with a rule that uses `SimpleLexiconFn`, such as the following: + + (rule $Relation ($VerbPrep) (SimpleLexiconFn (type (-> fb:type.any fb:type.any)))) + +This rule basically says that when we look for binaries, we shouldn't look at any phrase, we should restrict to phrases consisting +of verbs followed by prepositions. +Of course, there are other grammatical constructions that could give rise to a binary, and you would need to add new rules/categories for these. +Nevertheless, using filtering to narrow down to the spans which trigger certain rules is a very powerful tool. + +## Compositional semantic functions + +Compositional semantic functions are used to create larger derivations from +smaller ones. + +It is important to distinguish these semantic functions, which also work +compositionally, from the definition of the recursive definition of logical +forms in the previous section. One semantic function could easily construct a +logical form that involved multiple logical compositions, for example, going +from *city* (`fb:location.citytown`) to *largest city* (`(argmax 1 1 +(fb:type.object.type fb:location.citytown) fb:location.location.area)`). + +### IdentityFn + +`IdentityFn` takes a logical form and returns the same logical form. + +This is the most basic composition function, useful for refactoring grammar categories. +For example, suppose you have the following rule that parses things into entities. + + (rule $Entity ...) + +You can then write: + + (rule $ROOT ($Entity) (IdentityFn)) + +### JoinFn + +`JoinFn` can be used in a number of different ways. + +#### Joining binaries and unaries + +The first way in which `JoinFn` combines a binary and a unary is as follows. +Recall that each binary (e.g., `fb:people.person.place_of_birth`) has a head +(also called arg0 or return) and modifier (also called arg1 and argument). + +Just for clarity, the concrete edge in the knowledge graph is: + + head -- [fb:people.person.place_of_birth] --> modifier + +and the type of `fb:people.person.place_of_birth` is + + (-> modifier_type head_type) + +The way to think about a binary is not a function that returns the places of +births of people, but rather as a relation connecting people (in the head +position) with locations (in the modifier position). + +When applying `JoinFn`, the unary can be joined with either `arg0` or `arg1`, +and either the binary can come before the unary in the sentence or vice-versa. +Thus, there are four ways in which a binary may be joined with a unary. +These four possibilities are defined via the following two options: + + - Ordering: either `binary,unary` or `unary,binary` + - Position: `unaryCanBeArg0` or `unaryCanBeArg1` or both + +To illustrate the two most common settings for these options, +suppose we have the following setup: + + (rule $Entity (barack obama) (ConstantFn fb:en.barack_obama)) + (rule $Entity (honolulu) (ConstantFn fb:en.honolulu)) + (rule $Binary (birthplace) (ConstantFn fb:people.person.place_of_birth)) + +then we can write + + (rule $ROOT ($Binary $Entity) (JoinFn binary,unary unaryCanBeArg1)) + +These rules will parse the following utterance as follows: + + birthplace honolulu # (fb:people.person.place_of_birth fb:en.honolulu) + +This corresponds to standard forward application of a binary, and indeed we can +equivalently write: + + (rule $ROOT ($Relation $Entity) (JoinFn forward)) + +Alternatively, we can alter the arguments and write + + (rule $ROOT ($Entity $Relation) (JoinFn unary,binary unaryCanBeArg1)) + +or equivalently + + (rule $ROOT ($Entity $Relation) (JoinFn backward)) + +and this will parse the following into the same logical form: + + honolulu birthplace # (fb:people.person.place_of_birth fb:en.honolulu) + +If we use + + (rule $ROOT ($Relation $Entity) (JoinFn binary,unary unaryCanBeArg0)) + +we can parse: + + birthplace barack obama # (!fb:people.person.place_of_birth fb:en.barack_obama) + +#### Macro substitution + +The second way to use `JoinFn` is as macro substitution. Simply add the +`betaReduce` flag. For example: + + (rule $LambdaRelation (birthplace) (ConstantFn (lambda x (!fb:people.person.place_of_birth (var x))))) + (rule $ROOT ($LambdaRelation $Entity) (JoinFn forward betaReduce)) + +This will parse: + + birthplace barack obama # (!fb:people.person.place_of_birth fb:en.barack_obama) + +#### Multi-argument macro substitution + +So far, `JoinFn` takes two arguments, but we can also specify one of the arguments: + + (rule $ROOT (birthplace $Entity) (JoinFn forward betaReduce (arg0 (lambda x (!fb:people.person.place_of_birth (var x)))))) + +This rule also parses *birthplace barack obama* in the expected way. An +equivalent and much clearer way to write this is the following, where we omit +`JoinFn` completely: + + (rule $ROOT (birthplace $Entity) (lambda x (!fb:people.person.place_of_birth (var x)))) + +Here is another example that takes multiple arguments and forms the set +containing two entities: + + (rule $ROOT ($Entity and $Entity) (lambda x (lambda y (or (var x) (var y))))) + +Going forward, you are encouraged to use this last form since it is most +transparent. In the future, it might even be backed by another `SemanticFn` +since `JoinFn` is getting a bit out of hand. + +### MergeFn + +`MergeFn` simply constructs a `MergeFormula`. For example, consider the +following rules: + + (rule $Set (female) (ConstantFn (fb:people.person.gender fb:en.female))) + (rule $Set (scientist) (ConstantFn (fb:people.person.profession fb:en.scientist))) + (rule $Set ($Set $Set) (MergeFn and)) + +We could then parse: + + female scientist # (and (fb:people.person.gender fb:en.female) (fb:people.person.profession fb:en.scientist)) + +We can also use the following rule + + (rule $Set ($Set or $Set) (MergeFn or)) + +to do set union: + + female or scientist # (or (fb:people.person.gender fb:en.female) (fb:people.person.profession fb:en.scientist)) + +### SelectFn + +`SelectFn` acts as a utility composition function that aids in refactoring grammars. +More specifically, it can be used to skip over certain categories or parts of an utterance (e.g., stop words) in a controlled manner. +For example, in a question-answering setup, you may have many utterances that +start with the word "what" (or "who" etc.), but suppose this word does not +really convey any semantic content (this is not quite true). +To handle this, + + (rule $Wh (what) (ConstantFn null)) + (rule $Wh (who) (ConstantFn null)) + (rule $ROOT ($Wh $Set) (SelectFn 1)) + +This rules allow us to simply ignore the presence of *what* or *who*. + +### Freebase-specific semantic functions + +There are two semantic functions, `freebase.LexiconFn` and `freebase.BridgeFn` +which are used in our first sematic parsing applications, but they probably +should be avoided unless you're specifically doing Freebase QA. Even in that +case, the main thing you should think about is: + + (rule $Entity ($PHRASE) (LexiconFn fbsearch) + +which uses the Freebase Search API to look up entities. Be aware here that the +API will generously return many candidate entities for any string you give it, +so if you are getting too many results, you should use filtering to constrain +the number of spans you look at. + +### Context-specific semantic functions + +`FuzzyMatchFn` is used for matching words with a context-specific graph. +More documentation coming soon. + +#### ContextFn + +`ContextFn` generates logical forms from the context. +Concretely, an `Example` optionally has an associated `ContextValue`, which contains an ordered list of `n` `Exchange`s. +Each `Exchange` contains an utterance, a logical form (i.e., a `Formula`), and its denotation (i.e., a `Value`). + +A `ContextValue` thus represents information that is known prior to parsing the utterance in an `Example`. +For example, when running SEMPRE in interactive mode, the `ContextValue` +contains information about the last (`n`) utterance(s) that was/were parsed. +`ContextValue`s can also be provided in a dataset. + +`ContextFn` can be used to resolve anaphora. +For example, suppose you are running SEMPRE in interactive mode: + + User: where has barack obama lived + System: ((fb:people.person.places_lived fb:en.barack_obama) +The `ContextValue` of the interactive system now contains an `Exchange` corresponding to this utterance, logical form, and the computed denotation, +i.e. a list of cities including `fb:en.honolulu` and `fb:en.washington_dc`. +(Note that by default only one `Exchange`, specifically the most recent one, is stored in the system's `ContextValue`). + +Now, suppose that the you then say: `where was he born`. +Parsing this utterance requires resolving the anaphoric reference `he`, and `ContextFn` facilitates this. +Specifically, we would make the following rule: + + (rule $ContextEntity (he) (ContextFn (depth 0) (type fb:people.person))) + +This rule will trigger on the word `he` and retrieve the unary `fb:en.barack_obama` from the logical +formula in the `ContextValue` (since it has type `fb:people.person`). +The `depth` argument specifies how deep/tall of a logical form subtree you want to retrieve. +In the above example, a depth of 0 was specified since we wanted to retrieve a +single bare entity (i.e., a leaf). + +Alternatively, you could have said: `and which of those cities is the biggest?`. +Here, we would want to essentially "pull out" the entire logical form from the context. +Or more specifically, we want to pull out the join of the `fb:people.person.places_lived` binary and the `fb:en.barack_obama` entity. +To do this, we could make the following rule: + + (rule $ContextSetFromJoin (those cities) (ContextFn (depth 1) (type fb:places_lived))) + +This rule would trigger on the phrase `those cities` and retrieve the logical form `(fb:people.person.places_lived fb:en.barack_obama)`. +Note that compared to the previous example, we changed both the type and the depth arguments. +Here, a depth of 1 indicated that we want a subtree of depth 1 (i.e., a tree with one level before the leaves), +which corresponds, for example, to a logical form resulting from a single join. + +# Grammar + +So far, we have talked about grammars as being a set of rules. But SEMPRE +`Grammar`s have additional supporting functionality which make it easier to +use. A grammar is specified in a `.grammar` file and is loaded by the +command-line flag `-Grammar.inPaths`. + +## Binarization + +First, the `Grammar` performs automatic binarization of the grammar, so if you have a rule: + + (rule $ROOT ($Subject $Verb $Object) ...) + +This rule gets converted to a binarized grammar where each right-hand-side has at most two elements: + + (rule $ROOT ($Subject $Intermediate) ...) + (rule $Intermediate (Verb $Object) ...) + +The consequence is that the grammar you write will not be exactly the same as +the grammar that the parser receives. + +## Macros + +If you find yourself writing a lot of repeated elements: + + (rule (person) (ConstantFn (fb:type.object.type fb:people.person))) + (rule (place) (ConstantFn (fb:type.object.type fb:location.location))) + (rule (location) (ConstantFn (fb:type.object.type fb:location.location))) + +then you might want to use macros (which are all prefixed with `@`): + + (def @type fb:type.object.type) + (def @person (@type fb:people.person)) + (def @place (@type fb:location.location)) + +so that you can simply write the three rules as follows: + + (rule (person) (ConstantFn @person)) + (rule (place) (ConstantFn @location)) + (rule (location) (ConstantFn @location)) + +The general form for defining a macro: + + (def @ ) + +Note that macros cannot take arguments. + +## For loops + +If you are defining multiple rules, with small variations, then you can use for +loops to create many rules. For example: + + (for @x (place location) + (rule (@x) (ConstantFn (fb:type.object.type fb:location.location))) + ) + +which is equivalent to: + + (rule (place) (ConstantFn (fb:type.object.type fb:location.location))) + (rule (location) (ConstantFn (fb:type.object.type fb:location.location))) + +The general form is: + + (for @ ( ... ) + + ... + + ) + +## Conditionals + +Grammar files do not have full conditionals (the point is not to develop a +full-blown programming language here). But if you want a grammar which is +parametrized by complexity, then you can use conditionals to include or exclude +statements: + + (when compositional + (rule $Set ($Set $Set) (MergeFn and)) + ) + + (when (not compositional) + (rule $Set ($Entity $Entity) (MergeFn and)) + ) + +To enable the first rule (and disable the second), you would pass +`-Grammar.tags compositional` on the command-line. + +In general: + + (when + + ... + + ) + +where is one of the following + + + (not ) + (and ) + +## Includes + +You can use the statement: + + (include ) + +to break up your grammar file into multiple files. It is not advisable to use +many small files; rather use the `when` construction. + +# Types + +SEMPRE uses a type system to associate each logical form with a type, which is +used to rule out obviously bad logical forms (e.g., `(+ fb:en.barack_obama 3)` +or `(fb:location.location.containedby fb:en.barack_obama)`). + +Types are in a sense redundant with the grammar categories (we could very well +use `$Int` to capture only logical forms that compute integers. But there are +a few differences: + +- Categories could be based on how the natural language, whereas types are a + pure function of the logical forms (e.g., `$Preposition` and `$Verb` might + both generate logical forms that map to the type `(-> fb:type.any + fb:type.any)`. + +- The parser can handle categories efficiently (e.g., for coarse-to-fine + parsing); it is not necessary to compute the logical form. Types only exist + with logical forms. + +Each type is an instance of `SemType`. + +## Atomic types (`AtomicSemType`) + +All the **atomic types** are related via a partial ordering (the subtype +relation), which is easiest to think of as a tree structure: + + | fb:type.any + | | fb:type.boolean + | | fb:type.number + | | | fb:type.int + | | | fb:type.float + | | fb:type.datetime + | | fb:type.text + | | fb:common.topic + | | | fb:people.person + | | | fb:location.location + | | | ... + +Atomic types can be combined to create higher-order types: + +## Pair types (`FuncSemType`) + +A **pair type** looks like this: + + (-> ) + +and is generally used to represent a binary relation, such as +`fb:people.person.place_of_birth`: + + (-> fb:location.location fb:people.person) + +This type might seem a bit backwards, but it is important to remember that we +should think of binaries not as functions but as relations. + +In fact, the pair type looks like a function, but really behaves like a pair +type (this is more appropriate for database queries and makes it easier to +reason about). + +## Union types (`UnionSemType`) + +A **union type** represent a disjunction over many types: + + (union ... ) + +Note that `(union fb:type.int fb:type.float)` is a supertype of `fb:type.int`. + +## Special types + +- Top type (top): corresponds to the most general type (including all atomic + and function types). + +- Bottom type (bot): corresponds to type failure. + +## Other notes + +Some of these types are inherited from Freebase (e.g., `fb:type.float`), but +others are SEMPRE-specific and made in the spirit of the Freebase names. +Identifiers begin with `fb:` even if you are working with an application which +has little to do with Freebase, just for notational compatibility. + +Types are assigned to logical forms via type inference (class `TypeInference` +in SEMPRE), and are importantly not part of the logical form itself. This +gives us the flexibility of experimenting with different type systems without +actually changing the meaning. Type inference in general is computationally +expensive, so the default implementation of `TypeInference` sometimes just +punt. + +The main operation one can perform with two types is to compute their *meet*, +which is the most general type which is the superset of both types. For example: + + (-> fb:location.location fb:type.any) meet (-> top fb:people.person) + +is + + (-> fb:location.location fb:people.person) + +# Running SEMPRE + +The main entry point to SEMPRE is `run`. Run is based on the execrunner Ruby +library from fig, which provides a small domain-specific language for +generating command-line options. (See `fig/lib/execrunner.rb` for more +documentation). + + ./run @mode= ... + +The mode specifies the specific way you want to use SEMPRE. + +## Execution directories + +When you run something like `./run @mode=freebase ...`, an execution directory +will be created in + + state/execs/.exec, + +where is a unique identifier. This directory contains the stdout in the +`log` file, as well as the grammar and the parameters. + +## Web interface + +If you want to launch a demo, then you probably want to use the web interface, +which can be triggered by using the `-server` option. For example: + + ./run @mode=simple -interactive false -server true diff --git a/Makefile b/Makefile index 732bd13..a4fc941 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,38 @@ -NAME=sempre +BUILD_DEPS = libsempre/sempre-core.jar \ + libsempre/sempre-cache.jar \ + libsempre/sempre-freebase.jar \ + libsempre/sempre-fbalignment.jar \ + libsempre/sempre-paraphrase.jar \ + libsempre/sempre-corenlp.jar \ + libsempre/sempre-jungle.jar -default: $(NAME).jar -DEPS := $(shell ls lib/*.jar) $(shell find src -name "*.java") +default: module-classes $(BUILD_DEPS) -classes: $(DEPS) - mkdir -p classes - javac -d classes -cp 'lib/*' -Xlint:all `find src -name "*.java"` - touch classes +module-classes: + scripts/extract-module-classes.rb + +core: libsempre/sempre-core.jar +libsempre/sempre-core.jar: \ + $(shell ls src/edu/stanford/nlp/sempre/*.java) \ + $(shell ls src/edu/stanford/nlp/sempre/test/*.java) + cd src/edu/stanford/nlp/sempre && ant compile + +cache: libsempre/sempre-cache.jar +libsempre/sempre-cache.jar: \ + $(shell find src/edu/stanford/nlp/sempre/cache -name "*.java") + cd src/edu/stanford/nlp/sempre/cache && ant compile + +corenlp: libsempre/sempre-corenlp.jar +libsempre/sempre-corenlp.jar: libsempre/sempre-core.jar libsempre/sempre-cache.jar \ + $(shell find src/edu/stanford/nlp/sempre/corenlp -name "*.java") + cd src/edu/stanford/nlp/sempre/corenlp && ant compile + +freebase: libsempre/sempre-freebase.jar +libsempre/sempre-freebase.jar: libsempre/sempre-core.jar libsempre/sempre-cache.jar \ + $(shell find src/edu/stanford/nlp/sempre/freebase -name "*.java") + cd src/edu/stanford/nlp/sempre/freebase && ant compile -$(NAME).jar: classes - jar cf $(NAME).jar -C classes . - jar uf $(NAME).jar -C src . clean: - rm -rf classes $(NAME).jar + rm -rf classes libsempre diff --git a/QUICKSTART.md b/QUICKSTART.md deleted file mode 100644 index 1bc5342..0000000 --- a/QUICKSTART.md +++ /dev/null @@ -1,93 +0,0 @@ -This is a quickstart guide for recreating the EMNLP 2013 or ACL 2014 system. - -# Download the Dependencies - -These commands will download necessary resources (choose emnlp2013 or acl2014 or both): - - ./download-dependencies core - ./download-dependencies emnlp2013 - ./download-dependencies acl2014 - ./download-dependencies fullfreebase_vdb - -See `README.md` for other requirements SEMPRE depends on -(JDK 7 and Ruby). - -# Install the Database - -Freebase is stored in a database called virtuoso. These commands will -install a copy of it. Make sure to execute the `git checkout tags/v7.0.0` -to ensure you have a compatible version of virtuoso (instead of the most -recent version). - - # For Ubuntu, make sure these dependencies are installed - sudo apt-get install -y automake gawk gperf libtool bison flex libssl-dev - # Clone into sempre folder - git clone https://github.com/openlink/virtuoso-opensource - cd virtuoso-opensource - git checkout tags/v7.0.0 - ./autogen.sh - ./configure --prefix=$PWD/install - make - make install - cd .. - -# Start the Database - -This will start the virtuoso database on `localhost:3093` and import freebase: - - ./scripts/virtuoso start lib/freebase/93.exec/vdb 3093 - -# Running the System on New Questions - -First make sure you have compiled sempre by running `make`. - -Create a file called `testinput` that has your test questions in this format: - - (example (utterance "what states make up the midwest us?") (targetValues (description ""))) - (example (utterance "what is the capital of france?") (targetValues (description "Paris"))) - -Then run this command to test the default trained system on those two examples: - - ./sempre @mode=train \ - @domain=webquestions \ - @sparqlserver=localhost:3093 \ - @cacheserver=local \ - -Learner.maxTrainIters 0 \ - -Dataset.inPaths test:testinput \ - -Builder.inParamsPath lib/models/15.exec/params \ - -Grammar.inPaths lib/models/15.exec/grammar \ - -Dataset.readLispTreeFormat true - -This run should take about a minute or two. This will save the output to -`state/execs/$N.exec/log` where `$N` is some number. The current system should -get the first example wrong and the second one correct. - -Alternatively, you can launch an interactive shell to test out the system: - - ./sempre @mode=interact \ - @domain=webquestions \ - @sparqlserver=localhost:3093 \ - @cacheserver=local \ - @load=15\ - @executeTopOnly=0 - -# Training the EMNLP 2013/ACL 2014 System - -This command will train the EMNLP 2013 system on the WebQuestions dataset. It takes -a little over three days to complete. - - ./sempre @mode=train \ - @sparqlserver=localhost:3093 \ - @domain=webquestions \ - @cacheserver=local - -This command will train the ACL 2014 system on the WebQuestions dataset. It takes -about 24 hours to complete. - - ./parasempre @mode=train \ - @sparqlserver=localhost:3093 \ - @domain=webquestions \ - @cacheserver=local - -Alternatively, you can sanity check the system on the Free917 dataset by -setting `@domain=free917`. This should about an hour to complete. diff --git a/README.md b/README.md index bb46917..7cdf839 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,42 @@ -# SEMPRE: Semantic Parsing with Execution +# SEMPRE 2.0: Semantic Parsing with Execution -SEMPRE is a toolkit for training semantic parsers, which map natural language -utterances to denotations (answers) via intermediate logical forms. See -TUTORIAL.md for a walkthrough of the system. +## What is semantic parsing? -If you use this system, please cite: +A semantic parser maps natural language utterances into an intermediate logical +form, which is "executed" to produce a denotation that is useful for some task. + +A simple arithmetic task: + +- Utterance: *What is three plus four?* +- Logical form: `(+ 3 4)` +- Denotation: `7` + +A question answering task: + +- Utterance: *Where was Obama born?* +- Logical form: `(place_of_birth barack_obama)` +- Denotation: `Honolulu` + +A virtual travel agent task: + +- Utterance: *Show me flights to Montreal leaving tomorrow.* +- Logical form: `(and (type flight) (destination montreal) (departure_date 2014.12.09))` +- Denotation: `(list ...)` + +By parsing utterances into logical forms, we obtain a rich representation that +enables mucher deeper, context-aware understanding beyond the words. With the +rise of natural language interfaces, semantic parsers are becoming increasingly +more powerful and useful. + +## What is SEMPRE? + +SEMPRE is a toolkit that makes it easy to develop semantic parsers for new +tasks. The main paradigm is to learn a feature-rich discriminative semantic +parser from a set of utterance-denotation pairs. One can also quickly +prototype rule-based systems, learn from other forms of supervision, and +combine any of the above. + +If you use SEMPRE in your work, please cite: @inproceedings{berant2013freebase, author = {J. Berant and A. Chou and R. Frostig and P. Liang}, @@ -13,41 +45,108 @@ If you use this system, please cite: year = {2013}, } -# Requirements +SEMPRE has been used in the following papers: -SEMPRE depends on the following: +- J. Berant and A. Chou and R. Frostig and P. Liang. [Semantic parsing on + Freebase from question-answer + pairs](http://cs.stanford.edu/~pliang/papers/freebase-emnlp2013.pdf). EMNLP, + 2013. + This paper introduced SEMPRE 1.0, applied it to question answering on + Freebase, and created the WebQuestions dataset. The paper focuses on scaling + up semantic parsing via alignment and bridging, and does not talk about the + SEMPRE framework at all. To reproduce those results, check out SEMPRE 1.0. +- J. Berant and P. Liang. [Semantic Parsing via + Paraphrasing](http://cs.stanford.edu/~pliang/papers/paraphrasing-acl2014.pdf). + ACL, 2014. + This paper also used SEMPRE 1.0. The paraphrasing model is somewhat of a + offshoot, and does not use many of the core learning and parsing utiltiies in + SEMPRE. To reproduce those results, check out SEMPRE 1.0. -* Java 7 -* Ruby (version 1.8.7 or 1.9) -* [fig](https://github.com/percyliang/fig) -* [Google Guava](https://code.google.com/p/guava-libraries/) (version 14) -* [Jackson JSON Processor](http://wiki.fasterxml.com/JacksonHome) (version 2.2) -* [TestNG](http://testng.org/) (version 6.8) -* [Apache Lucene](http://lucene.apache.org/) (version 4.4) -* [Stanford CoreNLP](http://nlp.stanford.edu/software/corenlp.shtml) (version 3.2) +## Where do I go next? -Aside from Java and Ruby, remaining software dependencies are among -what's fetched by the `download-depenencies` script. +- If you're new to semantic parsing, you can learn more from the [background + reading section of the + tutorial](http://github.com/percyliang/sempre/blob/master/TUTORIAL.md). +- Install SEMPRE using the instructions under **Installation** below. +- Walk through the + [tutorial](http://github.com/percyliang/sempre/blob/master/TUTORIAL.md) + to get a hands-on introduction to semantic parsing through SEMPRE. +- Read the complete + [documentation](http://github.com/percyliang/sempre/blob/master/DOCUMENTATION.md) + to learn about the different components in SEMPRE. -# Tests +# Installation -To troubleshoot or check for repository health, you can run a suite of unit -tests that come packaged with the system. The command for this is: +## Requirements - ./sempre @mode=test -excludegroups xfail +You must have the following already installed on your system. -These tests assume you have a connection to a SPARQL endpoint at localhost:3093. -To avoid this assumption, you can exclude any tests that require a SPARQL -server by instead running: +- Java 8 (not 7) +- Ant 1.8.2 +- Ruby 1.8.7 or 1.9 - ./sempre @mode=test -excludegroups xfail,sparql +Other dependencies will be downloaded as you need them. SEMPRE has been tested +on Ubuntu Linux 12.04 and MacOS X. Your mileage will vary depending on how +similar your system is. -And if you don't have emnlp2013 dependencies downloaded, you'll need to exclude -yet another group of unit tests: +## Easy setup - ./sempre @mode=test -excludegroups xfail,sparql,emnlp2013 +1. Clone the GitHub repository: -All of these tests should pass. + git clone https://github.com/percyliang/sempre + +2. Download the minimal core dependencies (all dependencies will be placed in `lib`): + + ./pull-dependencies core + +3. Compile the source code (this produces `libsempre/sempre-core.jar`): + + make core + +4. Run an interactive shell: + + ./run @mode=simple + + You should be able to type the following into the shell and get the answer `(number 7)`: + + (execute (call + (number 3) (number 4))) + +To go further, check out the +[tutorial](http://github.com/percyliang/sempre/blob/master/TUTORIAL.md). + +## Virtuoso graph database + +If you will be using natural language to query databases (e.g., Freebase), then +you will also need to setup your own Virtuoso database (unless someone already +has done this for you): + + # For Ubuntu, make sure these dependencies are installed + sudo apt-get install -y automake gawk gperf libtool bison flex libssl-dev + + # Clone the repository + git clone https://github.com/openlink/virtuoso-opensource + cd virtuoso-opensource + git checkout tags/v7.0.0 + + # Configure + ./autogen.sh + mv INSTALL INSTALL.txt # Avoid conflict on case-insensitive file systems + ./configure --prefix=$PWD/install + + # Make (this takes a while) + make + make install + cd .. + +# ChangeLog + +Changes from SEMPRE 1.0 to SEMPRE 2.0: + +- Updated tutorial and documentation. +- Refactored into a core part for building semantic parsers in general; + interacting with Freebase and Stanford CoreNLP are just different modules. +- Removed fbalignment (EMNLP 2013) and paraphrase (ACL 2014) components to + avoid confusion. If you want to reproduce those systems, use SEMPRE 1.0. # License diff --git a/TUTORIAL.md b/TUTORIAL.md index 03611d2..0b22417 100644 --- a/TUTORIAL.md +++ b/TUTORIAL.md @@ -1,50 +1,59 @@ -# SEMPRE: Semantic Parsing with Execution +# SEMPRE 2.0 tutorial In this tutorial, we will provide a brief tour of SEMPRE. This tutorial is very much about the mechanics of the system, not about the linguistics or semantic parsing from a research point of view (for those, see the recommended -readings at the end of this document). +readings at the end of this document). For the full documentation, see +[DOCUMENTATION.md](https://github.com/percyliang/sempre/blob/master/DOCUMENTATION.md). We will construct a semantic parser to understand a toy subset of natural -language. Concretely, the system will have the following behavior: +language. Concretely, the system we will build will have the following +behavior: - Input: *What is three plus four?* - Output: 7 -In semantic parsing, *natural language utterances* are mapped into *logical -forms* (think programs), which are executed to produce some *denotation* (think -return value). +Recall that in semantic parsing, *natural language utterances* are mapped into +*logical forms* (think programs), which are executed to produce some +*denotation* (think return value). -## Configuration +We have assumed you have already downloaded SEMPRE and can open up a shell: -To download all the dependencies for the system, run: - - ./download-dependencies core - -To compile the system, run: - - make - -To run the system, run: - - java -Xmx3g -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor JavaExecutor -interactive + ./run @mode=simple This will put you in an interactive prompt where you can develop a system and -parse utterances into tiny Java programs (hence `JavaExecutor`). Note: you -might find it convenient to use `rlwrap` to get readline support. +parse utterances into tiny Java programs. Note: you might find it convenient +to use `rlwrap` to get readline support. -## Formulas and denotations +Just to provide a bit of transparency: The `run` script simply creates a +shell command and executes it. To see which command is run, do: -A logical form (`Formula`) is a hierarchical expression. For example, here are -some primitive logical forms: + ./run @mode=simple -n + +This should print out: + + rlwrap java -cp libsempre/*:lib/* -ea edu.stanford.nlp.sempre.Main -interactive + +You can pass in additional options: + + ./run @mode=simple -Parser.verbose 3 # Turn on more verbose debugging for the parser + ./run @mode=simple -help # Shows all options and default values + +## Section 1: Logical forms and denotations + +A **logical form** (class `Formula` in SEMPRE) is a hierarchical expression. +In the base case, we have primitive logical forms for representing concrete +values (booleans, numbers, strings, dates, names, and lists): (boolean true) (number 3) - (string hello) - (date 2013 7 28) + (string "hello world") + (date 2014 12 8) + fb:en.barack_obama + (list (number 1) (number 2)) Logical forms can be constructed recursively using `call`, which takes a -function name followed by arguments: +function name followed by arguments, which are themselves logical forms: (call + (number 3) (number 4)) (call java.lang.Math.cos (number 0)) @@ -52,36 +61,68 @@ function name followed by arguments: (call .substring (string "what is this?") (number 5) (number 7)) (call if (call < (number 3) (number 4)) (string yes) (string no)) -Note that each logical form is painfully explicit about types. You would -probably not want to program directly in this language, but that is not the -point; we will generate these logical forms automatically from natural -language. +In general: -We can execute these logical forms by typing into the interactive prompt: + (call ... ) + +Later, we will see other ways (besides `call`) of building more complex logical +forms from simpler logical forms. + +Note that each logical form is painfully explicit about types. You would +probably not want to program directly in this language (baby Java using +LISP-like notation), but that is not the point; we will later generate these +logical forms automatically from natural language. + +We can execute a logical form by typing the following into the interactive +prompt: (execute (call + (number 3) (number 4))) -This should print out `(number 7)`, which we refer to as the *denotation* of -the logical form. Try to execute the other logical forms and see what you get. +In general: + + (execute ) + +This should print out `(number 7)`, which we refer to as the *denotation* +(class `Value` in SEMPRE) of the logical form. Try to execute the other +logical forms and see what you get. + +**Exercise 1.1**: write a logical form that computes the first word +("compositionality") in the string "compositionality is key". ### Lambda expressions -So far, we have been representing monolithic logical forms. When we start -doing parsing, it will be convenient to be able to refer to logical forms with -some of its parts abstracted out. For example, the following logical form -represents a function that takes a number and returns its square: +So far, we have been representing logical forms that produce a single output +value (e.g., "compositionality"). But one of the key ideas of having programs +(logical forms) is the power of abstraction — that we can represents +*functions* that compute an output value for each input value. + +For example, the following logical form denotes a function that takes a number +and returns its square: (lambda x (call * (var x) (var x))) -We apply a lambda expression in the usual Lispy way: +If you execute this logical form directly, you will get an error, because the +denotation of this logical form is a function, which is not handled by the +`JavaExecutor`. However, we can apply this function to an argument `(number +3)`: ((lambda x (call * (var x) (var x))) (number 3)) -Executing this expression should yield `(number 9)`. +This logical form now denotes a number. Executing this logical form should +yield `(number 9)`. -Technical note: these lambda expressions are just doing macro substitution, not -actually representing higher-order functions; since there are no side effects -here, there is no difference. +In general: + + ( ) + +**Exercise 1.2**: Adapt your logical form form Exercise 1.1 to compute the +first word of any string. Your answer should be `(lambda x ...)`. Create a +logical form that applies this on the argument `(string "compositionality is +key")`. + +Technical note: these lambda expressions are actually just doing macro +substitution, not actually representing higher-order functions; since there are +no side effects here, there is no difference. This concludes the section on logical forms and denotations. We have presented one system of logical forms, which are executed using `JavaExecutor`. The @@ -89,30 +130,32 @@ system supports other types of logical forms, for example, those which encode SPARQL database queries for question answering (we will get to that later). Note that there is no mention of natural language yet... -## Parsing +## Section 2: Parsing utterances to logical forms Having established the nature of logical forms and their denotations, let us turn to the problem of mapping a natural language utterance into a logical -form. We will proceed by defining a *grammar*, which is a set of rules -which specify how to perform this mapping piece by piece. This is a grammar -in the computer science sense, not in the linguistic sense (again, SEMPRE -just provides a framework; you can use whatever grammar you would like). +form. Again, the key framework is *compositionality*, which roughly says that +the meaning of a full sentence is created by combining the meanings of its +parts. For us, meanings are represented by logical forms. + +We will start by defining a **grammar** (class `Grammar` in SEMPRE), which is a +set of **rules** (class `Rule` in SEMPRE), which specify how to combine logical +forms to build more complex ones in a manner that is guided by the natural +language. We will run through some examples to give you a feel for how things work, and then go into the details. First, let us add a rule to the grammar by -entering the following into the prompt: +typing the following into the interactive prompt: (rule $ROOT (three) (ConstantFn (number 3))) -Now to parse an utterance, just type it in: +Now to parse an utterance, just type it in to the interactive prompt: three -Note: The first time you parse an utterance will be slower since Stanford CoreNLP -components need to be loaded into memory (e.g., the part-of-speech tagger). - The parser should print out (among other information) a line that shows that -the utterance was parsed sucessfully into a *Derivation*: +the utterance was parsed successfully into a **derivation** (class `Derivation` +in SEMPRE), which importantly carries the correct logical form `(number 3.0)`: (derivation (formula (number 3.0)) (value (number 3.0)) (type fb:type.number)) @@ -120,31 +163,32 @@ Now type in the utterance: four -You should get no results because no rule will match `four`. To fix that, let +You should get no results because no rule matches `four`. To fix that, let us create a more general rule: (rule $ROOT ($PHRASE) (NumberFn)) -This rule says match any phrase (sequence of tokens), pass it to a special -function called `NumberFn`, which will transform the string into a new -derivation. +This rule says for any phrase (sequence of consecutive tokens), pass it to a special +function called `NumberFn`, which will transform the phrase string into a new +derivation representing a number. -Now the system can interpret general numbers! Try typing in: +Now, you can parse the following: - twenty five million - -The result should be `(number 2.5E7)`. + four + 2.718 Note: if you now type in `three`, you should get two derivations that yield the -same answer, one coming from each rule. +same answer, one coming from each rule. Note that `twenty-five million` will +not parse because we are using `SimpleLanguageAnalyzer`. Later, we can using +Stanford CoreNLP to improve the basic linguistic capabilities. So far, we have only parsed utterances using one rule, but the true power of these grammars come from combining multiple rules. Copy and paste in the following rules: (rule $Expr ($PHRASE) (NumberFn)) - (rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) - (rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) + (rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))) + (rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))) (rule $Partial ($Operator $Expr) (JoinFn forward)) (rule $Expr ($Expr $Partial) (JoinFn backward)) (rule $ROOT ((what optional) (is optional) $Expr (? optional)) (IdentityFn)) @@ -153,7 +197,7 @@ Now try typing in: What is three plus four? -The output should be +The output should be: (number 7) @@ -168,15 +212,15 @@ operations anywhere. Hopefully that should give you a sense of what parsing looks like. Let us now take a closer look. At the end of the day, a grammar declaratively specifies a -mapping from utterances to a set of candidate *Derivation*s (which are to be -scored and ranked later). A parser is an actual algorithm that takes the -grammar and generates those derivations. Recall that a derivation looks like this: +mapping from utterances to a set of candidate derivations. A **parser** (class +`Parser` in SEMPRE) is an actual algorithm that takes the grammar and generates +those derivations. Recall that a derivation looks like this: (derivation (formula (number 3.0)) (value (number 3.0)) (type fb:type.number)) -Formally, each Derivation produced by the parser has the following properties: +Formally, each derivation produced by the parser has the following properties: -1. Span i:j (e.g., 0:1): specifies the continguous portion of the input +1. Span i:j (e.g., 0:1): specifies the contiguous portion of the input utterance (tokens i to j-1) that the Derivation is constructed from. 2. Category (e.g., `$ROOT`): categories place hard constraints on what Derivations can be combined. @@ -203,7 +247,7 @@ There are some special categories: Now let us see how a grammar specifies the set of derivations. A grammar is a set of rules, and each rule has the following form: - (rule |target| (|source_1| ... |source_k|) |semantic function|) + (rule ( ... ) ) 1. Target category (e.g., `$ROOT`): any derivation produced by this rule is labeled with this category. `$ROOT` is the designated top-level category. @@ -216,10 +260,12 @@ A grammar is a set of rules, and each rule has the following form: 3. Semantic function (`SemanticFn`): a semantic function takes a sequence of derivations corresponding to the categories in the children and produces a set of new derivations which are to be labeled with the target category. - Semantic functions are arbitrary Java code, and allow the parser to integrate - custom logic in a flexible modular way. In the example above `ConstantFn` - is an example of a semantic function which always returns one `Derivation` - with the given logical form (e.g., `(number 3)`). + Semantic functions run arbitrary Java code, and allow the parser to integrate + custom logic in a flexible modular way. In the example above, `ConstantFn` + is an example of a semantic function which always returns one derivation + with the given logical form (e.g., `(number 3)`). `JoinFn` produces a + derivation whose logical form is the composition of the logical forms of the + two source derivations. Derivations are built recursively: for each category and span, we construct a set of Derivations. We can apply a rule if there is some segmentation of the @@ -228,8 +274,8 @@ with category |source_i|. In this case, we pass the list of derivations as input into the semantic function. The output is a set of derivations (possibly zero). -The first rule is a familiar one that just parses numbers such as `three -million` into the category `$Expr`: +The first rule is a familiar one that just parses strings such as *three* +into the category `$Expr`: (rule $Expr ($PHRASE) (NumberFn)) @@ -241,22 +287,8 @@ category `$Expr` and span 0:1. The same goes for *four* on span 2:3. The next two rules map the tokens *plus* and *times* to a static logical form (returned by `ConstantFn`): - (rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) - (rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) - -Note that the `ConstantFn` specifies both the logical form and the type (note -that the type is a property of the derivation, not of the logical form). Here, -the logical form is - - (lambda y (lambda x (call + (var x) (var y)))) - -which represents a curried function that takes `y` and `x` and returns calls -`+` on the values. The type of this entry is: - - (-> fb:type.number (-> fb:type.number fb:type.number)) - -The type is used in parsing to prevent bad derivations from being combined -spuriously. + (rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))) + (rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))) The next two rules are the main composition rules: @@ -266,16 +298,16 @@ The next two rules are the main composition rules: The semantic function `(JoinFn forward)` takes two a lambda term `$Operator` and an argument `$Expr` and returns a new derivation by forward application: - Input $Operator: (lambda y (lambda x (call + (var x) (var y)))) - Input $Expr: (number 4) - Output $Partial: (lambda x (call + (var x) (number 4))) + Source $Operator: (lambda y (lambda x (call + (var x) (var y)))) + Source $Expr: (number 4) + Target $Partial: (lambda x (call + (var x) (number 4))) The semantic function `(Join backward)` takes an argument `$Expr` and a lambda term `$Partial` and returns a new derivation by backward application: - Input $Expr: (number 3) - Input $Partial: (lambda x (call + (var x) (number 4))) - Output $Expr: (call + (number 3) (number 4)) + Source $Expr: (number 3) + Source $Partial: (lambda x (call + (var x) (number 4))) + Target $Expr: (call + (number 3) (number 4)) (rule $ROOT ((what optional) (is optional) $Expr (? optional)) (IdentityFn)) @@ -283,80 +315,233 @@ We allow some RHS elements to be optional, so that we could have typed in `three plus four` or `three plus four?`. `IdentityFn` simply takes the logical form corresponding to `$Expr` and passes it up. -## Learning +The complete derivation for *three plus four* is illustrated here: + + $ROOT : (call + (number 3) (number 4))) + | [IdentityFn] + $Expr : (call + (number 3) (number 4))) + | [JoinFn backward] + +-------------------------+-------------------------+ + | | + | $Partial : (lambda x (call + (var x) (number 4))) + | | [JoinFn forward] + | +-----------------------------+------------------------------+ + | | | + $Expr : (number 3) $Operator : (lambda y (lambda x (call + (var x) (var y)))) $Expr : (number 4) + | [NumberFn] | [ConstantFn] | [NumberFn] + $PHRASE: three | $PHRASE : four + | [built-in] | | [built-in] + three plus four + + +**Exercise 2.1**: write rules can parse the following utterances into +into the category `$Expr`: + + length of hello world # 11 + length of one # 3 + +Your rules should look something like: + + (rule $Function (length of) ...) + (rule $Expr ($Function $PHRASE) ...) + +**Exercise 2.2**: turn your "first word" program into a rule so that you can +parse the following utterances into `$String`: + + first word in compositionality is key # compositionality + first word in a b c d e # a + +**Exercise 2.3**: combine all the rules that you have written to produce one grammar +that can parse the following: + + two times length of hello world # 22 + length of hello world times two # (what happens here?) + +To summarize, we have shown how to connect natural language utterances and +logical forms using grammars, which specify how one can compositionally form +the logical form incrementally starting from the words in the utterance. Note +that we are dealing with grammars in the computer science sense, not in the +linguistic sense, as we are not developing a linguistic theory of +grammaticality; we are merely trying to parse some useful subset of utterances +for some task. Given an utterance, the grammar defines an entire set of +derivations, which reflect both the intrinsic ambiguity of language as well as +the imperfection of the grammar. In the next section, we will show how to +learn a semantic parser that can resolve these ambiguities. + +### Saving to a file (optional) + +You can put a set of grammar rules in a file (e.g., +`data/tutorial-arithmetic.grammar`) and load it: + + ./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar + +If you make edit the grammar, you can reload the grammar without exiting the +prompt by typing: + + (reload) + +### Using CoreNLP (optional) + +Recall that we were able to parse *four*, but not *twenty-five million*, +because we used the `SimpleLanguageAnalyzer`. In this section, we will show +how to leverage Stanford CoreNLP, which provides us with more sophisticated +linguistic processing on which we can build more advanced semantic parsers. + +First, we need to do download an additional dependency (this could take a while +to download because it loads all of the Stanford CoreNLP models for +part-of-speech tagging, named-entity recognition, syntactic dependency parsing, +etc.): + + ./pull-dependencies corenlp + +Compile it: + + make corenlp + +Now we can load the SEMPRE interactive shell with `CoreNLPAnalyzer`: + + ./run @mode=simple -languageAnalyzer corenlp.CoreNLPAnalyzer -Grammar.inPaths data/tutorial-arithmetic.grammar + +The following utterances should work now (the initial utterance will take a few +seconds while CoreNLP models are being loaded): + + twenty-five million + twenty-five million plus forty-two + +## Section 3: Learning So far, we have used the grammar to generate a set of derivations given an -utterance. In general, this set is huge and we need a way to rank the -derivations. Our strategy is to provide the system with training examples, -from which the system can learn a model that places a distribution over -derivations given utterances. +utterance. We could work really hard to make the grammar not overgenerate, but +this will in general be hard to do without tons of manual effort. So instead, we will +use machine learning to learn a model that can choose the best derivation (and +thus logical form) given this large set of candidates. So the philosophy is: -To enable learning in the interactive prompt, do the following (we could have -also passed these in as command-line arguments, e.g., -`-Master.onlineLearnExamples true -FeatureExtractor.featureDomains rule`): +- Grammar: small set of manual rules, defines the candidate derivations +- Learning: automatically learn to pick the correct derivation using features - (set Master.onlineLearnExamples true) - (set FeatureExtractor.featureDomains rule) +In a nutshell, the learning algorithm (class `Learner` in SEMPRE) uses +stochastic gradient descent to optimize the conditional log-likelihood of the +denotations given the utterances in a training set. Let us unpack this. -The first statement turns on online learning updates (this is only necessary -for the interactive prompt). The second statement adds features that keep -track of which rules we are using. +### Components of learning + +First, for each derivation, we extract a set of **features** (formally a map +from strings to doubles — 0 or 1 for indicator features) using a feature +extractor (class `FeatureExtractor` in SEMPRE), which is an arbitrary function +on the derivation. Given a parameter vector theta (class `Params` in SEMPRE), +which is also a map from strings to doubles, the inner product gives us a +score: + + Score(x, d) = features(x, d) dot theta, + +where x is the utterance and d is a candidate derivation. + +Second, we define a **compatibility function** (class `ValueEvaluator` in SEMPRE) +between denotations, which returns a number between 0 and 1. This allows us +learn with approximate values (e.g., "3.5 meters" versus "3.6 meters") and +award partial credit. + +Third, we have a dataset (class `Dataset` in SEMPRE) consisting of **examples** +(class `Example` in SEMPRE), which specifies utterance-denotation pairs. +Datasets can be loaded from files; here is what +`data/tutorial-arithmetic.grammar` looks like: + + (example + (utterance "three and four") + (targetValue (number 7)) + ) + +Intuitively, the learning algorithm will tune the parameter vector theta so +that derivations with logical forms whose denotations have high compatibility +with the target denotation are assigned higher scores. For the mathematical +details, see the learning section of this +[paper](http://www.stanford.edu/~cgpotts/manuscripts/liang-potts-semantics.pdf). + +### No learning As a simple example, imagine that a priori, we do not know what the word *and* means: it could be either plus or times. Let us add two rules to capture the -two possibilities: +two possibilities (this is reflected in `data/tutorial-arithmetic.grammar`): - (rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) (rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) + (rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) -Now type in +Start the interactive prompt: + + ./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar + +and type in: three and four -There should be two derivations each with probability 0.5: +There should be two derivations each with probability 0.5 (the system arbitrarily chooses one): - (derivation (formula (((lambda y (lambda x (call + (var x) (var y)))) (number 4.0)) (number 3.0))) (value (number 7.0)) (type fb:type.number)) [score=0, prob=0.500] (derivation (formula (((lambda y (lambda x (call * (var x) (var y)))) (number 4.0)) (number 3.0))) (value (number 12.0)) (type fb:type.number)) [score=0, prob=0.500] + (derivation (formula (((lambda y (lambda x (call + (var x) (var y)))) (number 4.0)) (number 3.0))) (value (number 7.0)) (type fb:type.number)) [score=0, prob=0.500] -You will also see the features that are active for the first derivation. For -example, the following feature represents the fact that one of the rules we -just added is active: +### Batch learning - [ rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))) ] 0 = 1 * 0 +To perform (batch) learning, we run SEMPRE: -So far all the features have weight zero, so all the scores (dot products -between feature vector and weight vector) are also zero. + ./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar -FeatureExtractor.featureDomains rule -Dataset.inPaths train:data/tutorial-arithmetic.examples -Learner.maxTrainIters 3 -We can add a training example which says that the utterance should map to the first derivation: +The `rule` feature domain tells the feature extractor to increment the feature +each time the grammar rule is applied in the derivation. `Dataset.inPaths` +specifies the examples file to train on, and `-Learner.maxTrainIters 3` +specifies that we will iterate over all the examples three times. - (accept 0) - -This should perform a stochastic gradient update of the parameters. Now type in: +Now type: three and four -The first derivation should have much higher probability (around 0.88). To see -the new parameters, type in: +The correct derivation should now have much higher score and probability: + + (derivation (formula (((lambda y (lambda x (call + (var x) (var y)))) (number 4)) (number 3))) (value (number 7)) (type fb:type.any)) [score=18.664, prob=0.941] + (derivation (formula (((lambda y (lambda x (call * (var x) (var y)))) (number 4)) (number 3))) (value (number 12)) (type fb:type.any)) [score=15.898, prob=0.059] + +You will also see the features that are active for the predicted derivation. +For example, the following line represents the feature indicating that we +applied the rule mapping *and* to `+`: + + [ rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call + (var x) (var y))))) ] 1.383 = 1 * 1.383 + +The feature value is 1, the feature weight is 1.383, and their product is the +additive contribution to the score of this derivation. You can look at the score of the other derivation: + + (select 1) + +The corresponding feature there is: + + [ rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call * (var x) (var y))))) ] -1.383 = 1 * -1.383 + +This negative contribution to the score is why we favored the `+` derivation +over this `*` one. + +We can also inspect the parameters: (params) -This concludes the basics of logical forms, denotations, grammars, features, -and learning. In practice, much of these operations can be done offline by -putting the grammar rules in a file (`data/tutorial.grammar`) and the examples -in another file (`data/tutorial.examples.json). Then you can one run command -to perform what we just did: +### Online learning - java -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor JavaExecutor -Grammar.inPaths data/tutorial.grammar -Dataset.inPaths train:data/tutorial.examples.json -readLispTreeFormat false -trainFrac 0.5 -devFrac 0.5 -maxTrainIters 2 -featureDomains rule -execDir tutorial.out +Finally, you can also do (online) learning directly in the prompt: -This will create a directory `tutorial.out` which records all the information -associated with this run. + (accept 1) -## Lambda DCS and SPARQL +This will accept the `*` derivation as the correct one and update the +parameters on the fly. If you type: -So far, we have worked with `JavaExecutor` as the engine that maps logical -forms to denotations by executing Java code. A major application of semantic -parsing is where the logical forms are database queries. In this section, we -will look at querying graph databases. + three and four + +again, you will see that the probability of the `+` derivation has decreased. +If you type `(accept 1)` a few more times, the `*` derivation will dominate +once more. + +## Section 4: Lambda DCS and SPARQL + +So far, we used `JavaExecutor` to map logical forms to denotations by executing +Java code. A major application of semantic parsing (and indeed the initial one +that gave birth to SEMPRE) is where the logical forms are database queries. In +this section, we will look at querying graph databases. A graph database (e.g., Freebase) stores information about entities and their properties; concretely, it is just a set of triples $(s, p, o)$, @@ -370,34 +555,65 @@ each triple is a directed edge between two nodes labeled with the property. See `data/tutorial.ttl` for an example of a tiny subset of the Freebase graph pertaining to geography about California. -We will use Virtuoso to provide the backend for querying this data. Make sure -you have Virtuoso downloaded and compiled (see QUICKSTART.md for instructions). +First, pull the dependencies needed for Freebase: -Start the server: + ./pull-dependencies freebase - scripts/virtuoso start tutorial.vdb 3001 +### Setting up your own Virtuoso graph database -Add the small graph to the database: +We use the graph database engine, Virtuoso, to store these triples and allow +querying. Follow these instructions if you want to create your own Virtuoso instance. - scripts/virtuoso add data/tutorial.ttl 3001 +First, make sure you have Virtuoso installed (see Installation section of the +README.md). -Aside: if you want to work with a larger database: +Then start the server: - ./download-dependencies geofreebase_ttl # Subset of Freebase for geography in .ttl format (not needed to run the server) - ./download-dependencies geofreebase_vdb # Virtuoso index for subset of Freebase for geography - ./download-dependencies fullfreebase_ttl # All of Freebase in .ttl format (not needed to run the server) [BIG FILE] - ./download-dependencies fullfreebase_vdb # Virtuoso index for all of Freebase [BIG FILE] + freebase/scripts/virtuoso start tutorial.vdb 3001 -Then, just replace `tutorial.vdb` with the appropriate `lib/freebase/??.exec/vdb` path. +Add a small graph to the database: -Now we are ready to make some queries. Start up the interactive prompt, now with the `SparqlExecutor`: + freebase/scripts/virtuoso add freebase/data/tutorial.ttl 3001 - java -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor SparqlExecutor -endpointUrl http://localhost:3001/sparql -interactive +Now you can query the graph (this should print out three items): -SPARQL is a language for querying these graphs, but it will be convenient to -use a language more tailored for semantic parsing which is based on a mix + ./run @mode=query @sparqlserver=localhost:3001 -formula '(fb:location.location.containedby fb:en.california)' + +To stop the server: + + freebase/scripts/virtuoso stop 3001 + +### Setting up a copy of Freebase + +The best case is someone already installed Freebase for you and handed you a +host:port. Otherwise, to run your own copy of the Freebase graph (a +2013 snapshot), read on. + +Download it (this is really big and takes a LONG time): + + ./pull-dependencies fullfreebase-vdb + +Then you can start the server (make sure you have at least 60GB of memory): + + freebase/scripts/virtuoso start lib/fb_data/93.exec/vdb 3093 + +### Lambda DCS + +SPARQL is the standard language for querying graph databases, but it will be +convenient to use a language more tailored for semantic parsing. We will use +[lambda DCS](http://arxiv.org/pdf/1309.4408.pdf), which is based on a mix between lambda calculus, description logic, and dependency-based compositional -semantics. The simplest formula is a single entity: +semantics (DCS). + +We assume you have started the Virtuoso database: + + freebase/scripts/virtuoso start tutorial.vdb 3001 + +Then start up a prompt: + + ./run @mode=simple-freebase @sparqlserver=localhost:3001 + +The simplest logical formula in lambda DCS is a single entity: fb:en.california @@ -453,74 +669,10 @@ Here are the following types of logical forms: `x` is in the set denoted by `u` and `y` is the value taken on by variable `v`. -See `SparqlExecutorTest.java` for examples. +See `src/edu/stanford/nlp/sempre/freebase/test/SparqlExecutorTest.java` for +many more examples (which only work on the full Freebase). -### Parsing - -So far, we have described the denotations of logical forms for querying a graph -database. Now we focus on parsing natural language utterances into these -logical forms. - -The core challenge is at the lexical level: mapping natural language phrases -(e.g., *born in*) to logical predicates (e.g., -`fb:people.person.place_of_birth`). First, we need to download the files that -specify this mapping (roughly, the mappings were created by aligning Freebase -and ClueWeb, as described the EMNLP 2013 paper, but you can just take these -mappings as given for now): - - ./download-dependencies emnlp2013 - -The following commands will require you having a Virtuoso instance that has all of Freebase indexed. -If you do not have this set up, run the following two commands: - - ./download-dependencies fullfreebase_vdb # Virtuoso index for all of Freebase [BIG FILE] - - scripts/virtuoso start lib/freebase/93.exec/vdb 3093 # Starts server on localhost at port 3093 - -Now, restart the SEMPRE shell, pointing the SparqlExecutor to the Virtuoso -instance: - - java -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor SparqlExecutor -endpointUrl http://localhost:3093/sparql -interactive - -Now, we can add rules with semantic function `LexiconFn`: - - (rule $Entity ($PHRASE) (LexiconFn entity allowInexact)) # e.g., Ulm - (rule $Unary ($LEMMA_PHRASE) (LexiconFn unary)) # e.g., physicists - (rule $Binary ($PHRASE) (LexiconFn binary)) # e.g., born in - (rule $ROOT ($Entity) (IdentityFn)) - (rule $ROOT ($Unary) (IdentityFn)) - -LexiconFn takes logical forms which corresponds to strings (e.g., `(string physicist)`) -and converts them into Freebase logical forms (e.g., -`(fb:people.person.profession fb:en.physicist)`). - -Type the following utterances into the prompt: - - Ulm - physicists - -Each utterance should return a list of candidate entities. - -Now for compositionality, add the following rules, which will take two -logical forms and either perform an intersection or a join: - - (rule $Set ($Unary $PartialSet) (MergeFn and)) - (rule $PartialSet ($Binary $Entity) (JoinFn binary,unary unaryCanBeArg1)) - (rule $ROOT ($Set) (IdentityFn)) - -This allows us to type in: - - physicists born in Ulm - -You should get many candidates, but the correct answer should appear in the list: - - (derivation (formula (and (fb:people.person.profession fb:en.physicist) (!fb:location.location.people_born_here fb:en.ulm))) (value (list (name fb:en.albert_einstein "Albert Einstein"))) (type fb:people.person)) - -For an example of a more complex grammar, look at `data/emnlp2013.grammar`. - -## Exercises - -1. Convert the following natural language utterances into lambda-DCS logical forms: +**Exercise 4.1**: write lambda DCS logical forms for the following utterances: `city with the largest area` @@ -534,17 +686,140 @@ For an example of a more complex grammar, look at `data/emnlp2013.grammar`. `country with the most number of rivers` -You should familiarize yourself with the [Freebase schema](http://www.freebase.com/schema) to see which -predicates to use. +You should familiarize yourself with the [Freebase +schema](http://www.freebase.com/schema) to see which predicates to use. +Execute these on the full Freebase to find out the answer! -Execute these logical forms on the `geofreebase` subset to verify your answers. +### Parsing -2. Write a grammar that can parse the above utterances into a set of candidates -containing the true logical form you annotated above. Train a model (remember -to add features) so that the correct logical forms appear at the top of the -candidate list. +So far, we have described the denotations of logical forms for querying a graph +database. Now we focus on parsing natural language utterances into these +logical forms. -## Background reading +The core challenge is at the lexical level: mapping natural language phrases +(e.g., *born in*) to logical predicates (e.g., +`fb:people.person.place_of_birth`). It is useful to distinguish between two +types of lexical items: + +- Entities (e.g., `fb:en.barack_obama`): There are generally a huge number of + entities (Freebase has tens of millions). Often, string matching gets you + part of the way there (for example, *Obama* to `fb.en:barack_obama`), but + there is often quite a bit of ambiguity (Obama is also a city in Japan). + +- Non-entities (e.g., `fb:people.person.place_of_birth`), which include unary + and binary predicates: There are fewer of these, but string matching is + unlikely to get you very far. + +We could always add grammar rules like this: + + (rule $Entity (the golden state) (ConstantFn fb:en.california)) + (rule $Entity (california) (ConstantFn fb:en.california)) + +but grammars are supposed to be small, so this approach does not scale, so we +are not going to do this. +One way is to create a **lexicon**, which is a mapping from words to predicates +(see `freebase/data/tutorial-freebase.lexicon`), with entries like this: + + {"lexeme": "california", "formula": "fb:en.california"} + {"lexeme": "the golden state", "formula": "fb:en.california"} + {"lexeme": "cities", "formula": "(fb:type.object.type fb:location.citytown)"} + {"lexeme": "towns", "formula": "(fb:type.object.type fb:location.citytown)"} + {"lexeme": "in", "formula": "fb:location.location.containedby"} + {"lexeme": "located in", "formula": "fb:location.location.containedby"} + +Then we can add the following rules (see +`freebase/data/tutorial-freebase.grammar`): + + (rule $Unary ($PHRASE) (SimpleLexiconFn (type fb:type.any))) + (rule $Binary ($PHRASE) (SimpleLexiconFn (type (-> fb:type.any fb:type.any)))) + (rule $Set ($Unary) (IdentityFn)) + (rule $Set ($Unary $Set) (MergeFn and)) + (rule $Set ($Binary $Set) (JoinFn forward)) + (rule $ROOT ($Set) (IdentityFn)) + +The `SimpleLexiconFn` looks up the phrase and returns all formulas that have the given type. To check the type, use: + + (type fb:en.california) # fb:common.topic + (type fb:location.location.containedby) # (-> fb:type.any fb:type.any) + +`MergeFn` takes the two (unary) logical forms |u| and |v| (in this case, coming from `$Unary` and `$Set`), +and forms the intersection logical form `(and |u| |v|)`. + +`JoinFn` takes two logical forms (one binary and one unary) and returns the +logical form `(|b| |v|)`. Note that before we were using `JoinFn` as function +application. In lambda DCS, `JoinFn` produces an actual logical form that +corresponds to joining |b| and |v|. The two bear striking similarities, which +is the basis for the overloading. + +Now start the interactive prompt: + + ./run @mode=simple-freebase @sparqlserver=localhost:3001 -Grammar.inPaths freebase/data/tutorial-freebase.grammar -SimpleLexicon.inPaths freebase/data/tutorial-freebase.lexicon + +We should be able to parse the following utterances: + + california + the golden state + cities in the golden state + towns located in california + +In general, how does one create grammars? One good strategy is to start with a +single rule mapping the entire utterance to the final logical form. Then +decompose the rule into parts. For example, you might start with: + + (rule $ROOT (cities in california) (ConstantFn (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.california)))) + +Then you might factor it into two pieces, in order to generalize: + + (rule $ROOT (cities in $Entity) (lambda e (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby (var e))))) + (rule $Entity (california) (ConstantFn fb:en.california)) + +Note that in the first rule, we are writing `(lambda x ...)` directly. This +means, take the logical form for the source (`$Entity`) and substitute it in +for `x`. + +We can refactor the first rule: + + (rule $ROOT ($Unary in $Entity) (lambda x (and (var u) (fb:location.location.containedby (var e))))) + (rule $Unary (cities) (ConstantFn (fb:type.object.type fb:location.citytown))) + +and so on... + +**Exercise 4.2**: Write a grammar that can parse the utterances from Exercise +4.1 into a set of candidates containing the true logical form you annotated. +Of course you can trivially write one rule for each example, but try to +decompose the grammars as much as possible. This is what will permit +generalization. + +**Exercise 4.3**: Train a model so that the correct logical forms appear at the +top of the candidate list on the training examples. Remember to add features. + +## Debugging + +In the beginning, SEMPRE grammars can be difficult to debug. This is primarily +because everything is dynamic, which means that minor typos result in empty +results rather than errors. + +The first you should do is to check that you do not have typos. Then, try to +simplify your grammar as much as possible (comment things out) until you have +the smallest example that fails. Then you should turn on more debugging +output: + +Only derivations that reach `$ROOT` over the entire span of the sentence are +built. You can also turn on debugging to print out all intermediate +derivations so that you can see where something is failing: + + (set Parser.verbose 3) # or pass -Parser.verbose 3 on the command-line + +Often derivations fail because an intermediate combination does not type check. +This option will print out all combinations which are tried. You might find +that you are combining two logical forms in the wrong way: + + (set JoinFn.verbose 3) + (set JoinFn.showTypeCheckFailures true) + (set MergeFn.verbose 3) + (set MergeFn.showTypeCheckFailures true) + +## Appendix: Background reading So far this tutorial has provided a very operational view of semantic parsing based on SEMPRE. The following references provide a broader look at the area diff --git a/data/tutorial-arithmetic.examples b/data/tutorial-arithmetic.examples new file mode 100644 index 0000000..d15f06f --- /dev/null +++ b/data/tutorial-arithmetic.examples @@ -0,0 +1,4 @@ +(example + (utterance "three and four") + (targetValue (number 7)) +) diff --git a/data/tutorial.grammar b/data/tutorial-arithmetic.grammar similarity index 60% rename from data/tutorial.grammar rename to data/tutorial-arithmetic.grammar index 2bfd73a..43bb91a 100644 --- a/data/tutorial.grammar +++ b/data/tutorial-arithmetic.grammar @@ -1,10 +1,12 @@ +# Arithmetic example from TUTORIAL.md + (rule $Expr ($PHRASE) (NumberFn)) -(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) -(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) +(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))) +(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))) (rule $Partial ($Operator $Expr) (JoinFn forward)) (rule $Expr ($Expr $Partial) (JoinFn backward)) (rule $ROOT ((what optional) (is optional) $Expr (? optional)) (IdentityFn)) -# Ambiguity -(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) -(rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number)))) +# Example of ambiguity to demonstrate learning +(rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))) +(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))) diff --git a/data/tutorial.examples.json b/data/tutorial.examples.json deleted file mode 100644 index 5392c98..0000000 --- a/data/tutorial.examples.json +++ /dev/null @@ -1,11 +0,0 @@ -[ -{ - "utterance": "What is seven and thirty five?", - "targetValue": "(number 42)" -}, - -{ - "utterance": "What is ten and ten?", - "targetValue": "(number 20)" -} -] diff --git a/data/unittest-learn.examples.json b/data/unittest-learn.examples.json deleted file mode 100644 index f82c8d3..0000000 --- a/data/unittest-learn.examples.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - {"utterance": "American", "targetFormula": "(fb:people.person.nationality fb:en.united_states)"}, - {"utterance": "American born in Paris", "targetFormula": "(and (fb:people.person.nationality fb:en.united_states) (fb:people.person.place_of_birth fb:en.paris))"} -] diff --git a/demo-www/bullet.png b/demo-www/bullet.png new file mode 100644 index 0000000000000000000000000000000000000000..94434730d4d3b77a748cc54efd87762db898ba84 GIT binary patch literal 966 zcmeAS@N?(olHy`uVBq!ia0vp^5+KaM1|%Pp+x`GjEa{HEjtmSN`?>!lvI6;RN#5=* zKpF^sI`6IrQk(@Ik;M!QTpvJ~(P@pz43J=1fnz$52AjuF;j?rZ0|V0vPZ!4!kK=cz zdS?VXinQ(D`Td#N&a;bWx!k(awN|fdhk*E^j)*nZtQYl;ZrxFGGsq=eFIIC$kQRp$ zhf)W>fcT^0Nby|JsEd~(OxOBedX^Taw}?aSQbg&(|7j1-e5fxzH)m#Jee5E|qKB_v zFdeU#e_-OGvw(p)=9e^sBiD3Bo!K3aEuJ6!zUCjtfeGHnlARmh$BVf3N#&Fs?pB$T z9ICKg(a2%jdh7Z%*}iGzm6eEi;+h%#^>a}!_ z$R=+4(YO1!)~;2iru~V2Q!dCbyj#D_MQqEpk}b<$oGcamYCF-F{ch~dC!M8l4?eB9 zBE8KnI&jBCW4<1S?){Sv+8NEA`R(AocYQU{CI@Htv+B%JkSe^CyD{g1dzn>-?5ZrK zqMK(v&2f!mY;UYy%kb-D$+Jt%du6jfXA5yY(-!S|>Kb!g@zG8d=KImJyVmD-yM43j zyP;R?mU(?&ddt#lMlaMaPxR&gR+jcz(0>9Clar*-SDQui+Mn?NV00j(+aX|vzVW}ZXTRq# zs+g+i2gV#NV4j{SGOhpY?iHDx24X7{a(>BnPd#Ox?0iC6LOm1mo-26S#*S*}x~HI<{XBT?iavs2fP zE9;-XoO*UPv&@+@cb;!LWMj5a*0p%Dv&atCl*WbEdiJmUcvw=cH`TmJbCOSx+hgYC zv)0A2K2Vvk;9eJ_z3rZ`t3PfWUGYET-TT@9<$p2hi16(`{@LLyFafKUxJHyD7o{ea zr0S*s2?iqrLqlByBV9xD5JLkiBXcVQ3ta;XD+2>|o6BK94cZ_L&L9n$`FRQp=3+6IOO2BK4W0%7*#=BH$)RifEr7Gh)!RBdH! W3b99|ZHp671B0ilpUXO@geCx?UW$wW literal 0 HcmV?d00001 diff --git a/demo-www/index.html b/demo-www/index.html new file mode 100644 index 0000000..c12abce --- /dev/null +++ b/demo-www/index.html @@ -0,0 +1 @@ +Hello there. diff --git a/demo-www/main.css b/demo-www/main.css new file mode 100644 index 0000000..6c392de --- /dev/null +++ b/demo-www/main.css @@ -0,0 +1,64 @@ +a:link {text-decoration: none; color: 00008b} +a:visited {text-decoration: none} +a:active {text-decoration: none} +a:hover {text-decoration: none; color: red} +body {background-color: white; margin-left: 50px; font-family: sans-serif} + +.question {margin: 10px; font-size: 24px} +.ask {font-size: 24px} +.answer { + font-size: 20; + /*background-image: url(bullet.png); + background-repeat: no-repeat;*/ + padding-left: 24px; +} +.message {font-size: 15} +.description {margin-top: 50px; font-size: 18} +.details { + border-style: groove; + margin-top: 10px; + padding: 10px; + font-size: 13px; + margin-right: 50px +} + +.listHeader {font-size:18; font-weight:bold; color:brown; white-space:nowrap} +.nowrap {white-space:nowrap} +.bubble {width:300px} + +.word {font-size:16; color:darkgreen; font-style:italic} +.predicate {font-size:16; color:red} +.tag {font-size:14; color:black; white-space:nowrap} + +.correctButton {color:green} +.wrongButton {color:red} + +.lexicalResponse table {border-spacing:3px 0px; text-align:center} +table.predInfo td {padding:1px; text-align:left; font-size:14; white-space:nowrap} + +table.context {padding:5; border:1px dashed brown} + +.valueTable {border: 2px solid gray; padding: 2px} + +.groupResponse td {vertical-align:top; padding:1px} +.listResponse li {font-size:12} + +.error {color:red; font-weight:bold} + +/* Tooltip (http://psacake.com/web/jl.asp) */ +a.info { + position:relative; + z-index:24; + color:#000; + text-decoration:none +} +a.info span.tooltip {display: none} + +a.info:hover {z-index:25; background-color:#ff0} +a.info:hover span.tooltip { + display:block; + position:absolute; + border:1px solid #0cf; + background-color:#eff; color:#000; + top:2em; left:2em +} diff --git a/demo-www/main.js b/demo-www/main.js new file mode 100644 index 0000000..81b1c1b --- /dev/null +++ b/demo-www/main.js @@ -0,0 +1,2 @@ +function submit() { +} diff --git a/download-dependencies b/download-dependencies deleted file mode 100755 index 91c6ffb..0000000 --- a/download-dependencies +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/ruby - -$modules = [:core, - :emnlp2013, - :acl2014, - :geofreebase_vdb, - :geofreebase_ttl, - :fullfreebase_vdb, - :fullfreebase_ttl] - -def usage - puts "Usage: ./download-dependencies <#{$modules.join('|')}>" -end - -if ARGV.size == 0 - usage - exit 1 -end - -def run(command) - puts "RUNNING: #{command}" - if not system(command) - puts "FAILED: #{command}" - exit 1 - end -end - -def download(release, path, baseUrl='http://nlp.stanford.edu/software/sempre') - url = baseUrl + '/release-' + release.to_s - isDirectory = (path.end_with?('.exec') or path.end_with?('vdb') or - path.end_with?('free917') or path.end_with?('inexact') or path.end_with?('prolog')) # Superficial test for directory - if release != :code and !path.end_with?('.gz', '.tgz', '.jar') - path += '.tar' if isDirectory - path += '.bz2' - end - run("mkdir -p #{File.dirname(path)}") - run("wget -c '#{url}/#{path}' -O #{path}") - if release != :code and path.end_with?('.bz2') - if isDirectory - run("cd #{File.dirname(path)} && tar xjf #{File.basename(path)}") - else - run("bzip2 -fd #{path}") - end - end -end - -def downloadFromFileList(release) - files = [] - File.foreach('release-' + release.to_s + '.files') { |line| - file = line.sub(/#.*$/, '').sub(/^\s*/, '').sub(/\s*$/, '') - next if file.size == 0 - files << file - } - files.each { |path| - download(release, path) - } -end - -def main(which) - if not $modules.include?(which) - usage - exit 1 - end - downloadFromFileList(which) -end - -ARGV.each { |mod| - mod = mod.to_sym - main(mod) -} diff --git a/fig/bin/chunk b/fig/bin/chunk index c9bd917..377a545 100755 --- a/fig/bin/chunk +++ b/fig/bin/chunk @@ -1,9 +1,9 @@ #!/usr/bin/ruby -require File.dirname($0)+'/../lib/myutils' +require 'pathname' +require File.dirname(Pathname.new($0).realpath) + '/../lib/myutils' # Print out a chunk of a file. Specifically, print out the header lines. -# Use $file, $headerNumLines, $chunkSize, $indices, $printNumChunks, $verbose = extractArgs(:spec => [ ['file', String, nil, true, 'Big file to read'], ['headerNumLines', Fixnum, 0, false, 'Number of header lines to include'], diff --git a/fig/bin/qcreate b/fig/bin/qcreate index 83e90eb..84fc958 100755 --- a/fig/bin/qcreate +++ b/fig/bin/qcreate @@ -83,6 +83,7 @@ cmdFile = "#{execPath}/#{id}.sh" out = open(cmdFile, 'w') cmd = ARGV.map{|x| x =~ /[ "]/ ? x.inspect : x}.join(' ').gsub(/_OUTPATH_/, execPath) cmd += " > #{execPath}/stdout 2> #{execPath}/stderr" if qsub +out.puts "#!/bin/bash" out.puts "cd #{Dir.pwd}" out.puts cmd out.close @@ -91,7 +92,7 @@ puts cmd system "git rev-parse HEAD > #{execPath}/git-hash" if File.exists?('.git') if qsub - exec("qsub #{cmdFile} -o /dev/null -e /dev/null") + exec("qsub #{File.expand_path(cmdFile)} -o /dev/null -e /dev/null") else - exec("bash #{cmdFile}") + exec("bash #{File.expand_path(cmdFile)}") end diff --git a/freebase/README.md b/freebase/README.md new file mode 100644 index 0000000..ed6c463 --- /dev/null +++ b/freebase/README.md @@ -0,0 +1,3 @@ +This directory contains all the small data files and scripts relevant to +semantic parsing on Freebase. In particular, it contains grammars from the +EMNLP 2013 system and other examples. diff --git a/freebase/data/emnlp2013.grammar b/freebase/data/emnlp2013.grammar new file mode 100644 index 0000000..d688c53 --- /dev/null +++ b/freebase/data/emnlp2013.grammar @@ -0,0 +1,127 @@ +# Grammar used for the EMNLP 2013 submission. +# This grammar handles simple questions involving one entity, one binary, and +# at most one unary. +# Supports Webquestions and Free917. + +############################################################ +# Lexicon + +### POS filter: Use POS tags to decide which spans should trigger something. + +# Matching unaries (SimpleNounPhrase) +(rule $Noun ($LEMMA_TOKEN) (FilterPosTagFn token WRB WP NN NNS NNP NNPS)) +(rule $SimpleNounPhrase ($Noun) (ConcatFn " ")) +(rule $SimpleNounPhrase ($Noun $SimpleNounPhrase) (ConcatFn " ")) + +# Matching entities (NamedEntity): note - don't lemmatize +# Match an entity if it is a sequence of NE tags or NNP tags or of minimal length +(rule $NamedEntity ($PHRASE) (FilterNerSpanFn PERSON ORGANIZATION LOCATION MISC)) +(rule $NamedEntity ($PHRASE) (FilterPosTagFn span NNP)) + +# Allow length 1 spans to trigger entities in Free917, but not in WebQuestions. +(when webquestions + (rule $TokenSpan ($PHRASE) (FilterSpanLengthFn 2)) +) +(when free917 + (rule $TokenSpan ($PHRASE) (FilterSpanLengthFn 1)) + (rule $TokenSpan ($LEMMA_TOKEN) (FilterSpanLengthFn 1)) +) + +# Matching binaries (CompositeRel) +(rule $Verb ($LEMMA_TOKEN) (FilterPosTagFn token VB VBD VBN VBG VBP VBZ VBD-AUX)) +(rule $Particle ($LEMMA_TOKEN) (FilterPosTagFn token RP)) +(rule $Prep ($LEMMA_TOKEN) (FilterPosTagFn token IN TO)) +(rule $Adj ($LEMMA_TOKEN) (FilterPosTagFn token JJ)) + +(rule $Rel ($LEMMA_TOKEN) (FilterPosTagFn token NN NNS NNP NNPS VB VBD VBN VBG VBP VBZ IN VBD-AUX JJ)) +(rule $BaseRel ($Rel) (IdentityFn)) # parents +(rule $BaseRel ($Verb $SimpleNounPhrase) (ConcatFn " ")) +(rule $BaseRel ($Verb $Particle) (ConcatFn " ")) # grow up +(rule $CompositeRel ($BaseRel) (ConcatFn " ")) # grow up, parents +(rule $CompositeRel ($BaseRel $Prep) (ConcatFn " ")) # grow up in, parents of + +### Lexicon: call LexiconFn on the spans. + +(when inexact + (rule $Entity ($NamedEntity) (LexiconFn entity inexact)) + (rule $Entity ($TokenSpan) (LexiconFn entity inexact)) +) +(when exact + (rule $Entity ($NamedEntity) (LexiconFn entity exact)) + (rule $Entity ($TokenSpan) (LexiconFn entity exact)) +) +(when fbsearch + (rule $Entity ($NamedEntity) (LexiconFn entity fbsearch)) + (rule $Entity ($TokenSpan) (LexiconFn entity fbsearch)) +) +(when fbsearch0 + (rule $Entity ($NamedEntity) (LexiconFn entity fbsearch)) +) +(rule $Entity ($PHRASE) (DateFn)) # Example: 2002 +(rule $Unary ($SimpleNounPhrase) (LexiconFn unary)) # Example: "city" +(rule $Unary (how $Adj) (ConstantFn (fb:type.object.type fb:type.int) fb:type.int)) +(rule $Unary (how $Adj) (ConstantFn (fb:type.object.type fb:type.float) fb:type.float)) +(rule $Binary ($CompositeRel) (LexiconFn binary)) # Example: "capital", "developed" + +#### Lexicon: hard coded unaries +(rule $Unary (who) (ConstantFn (fb:type.object.type fb:people.person) fb:people.person)) +(rule $Unary (who) (ConstantFn (fb:type.object.type fb:organization.organization) fb:organization.organization)) +(rule $Unary (who) (ConstantFn (fb:type.object.type fb:business.company) fb:business.company)) +(rule $Unary (where) (ConstantFn (fb:type.object.type fb:organization.organization) fb:organization.organization)) +(rule $Unary (where) (ConstantFn (fb:type.object.type fb:location.location) fb:location.location)) +(rule $Unary (where) (ConstantFn (fb:type.object.type fb:education.educational_institution) fb:education.educational_institution)) +(rule $Unary (when) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime)) +(rule $Unary (date) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime)) +(rule $Unary (year) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime)) +(rule $Unary (day) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime)) + +### Lexicon: hard coded binaries +(rule $Binary (,) (ConstantFn fb:location.location.containedby (-> fb:location.location fb:location.location))) + +# Aggregation +(rule $CountStr (how many) (ConstantFn null null)) +(rule $CountStr (number of) (ConstantFn null null)) +(rule $Operator ($CountStr) (ConstantFn (lambda x (count (var x))) (-> fb:common.topic fb:type.number))) + +############################################################ + +(rule $Padding ($PHRASE) (IdentityFn)) # Can skip these sequences + +(rule $ROOT (($Padding optional) $Operator ($Padding optional) $Set) (JoinFn forward betaReduce)) +(rule $ROOT (($Padding optional) $Set ($Padding optional)) (IdentityFn)) + +### Combine entity and binary +(when join + (rule $BaseSet ($Entity ($Padding optional) $Binary) (JoinFn unary,binary unaryCanBeArg0 unaryCanBeArg1)) + (rule $BaseSet ($Binary ($Padding optional) $Entity) (JoinFn binary,unary unaryCanBeArg0 unaryCanBeArg1)) +) + +### Create binary out of thin air +(when bridge + (rule $BaseSet ($Unary ($Padding optional) $Entity) (BridgeFn unary headFirst)) + (rule $BaseSet ($Entity ($Padding optional) $Unary) (BridgeFn unary headLast)) + (rule $BaseSet ($Entity) (BridgeFn entity headLast)) +) + +(when inject + (rule $BaseSet ($BaseSet ($Padding optional) $Entity) (BridgeFn inject headFirst)) + (rule $BaseSet ($Entity ($Padding optional) $BaseSet) (BridgeFn inject headLast)) +) + +(when (not compositional) + (rule $Set ($BaseSet) (IdentityFn)) + (rule $Set ($Unary ($Padding optional) $BaseSet) (MergeFn and)) +) + +(when compositional + (rule $Set ($BaseSet) (IdentityFn)) + + ### Combine set and binary + (when join + (rule $Set ($Binary ($Padding optional) $Set) (JoinFn binary,unary unaryCanBeArg0 unaryCanBeArg1)) + (rule $Set ($Set ($Padding optional) $Binary) (JoinFn unary,binary unaryCanBeArg0 unaryCanBeArg1)) + ) + + # Merge two sets + (rule $Set ($Set ($Padding optional) $Set) (MergeFn and)) +) diff --git a/freebase/data/free917.test.examples.canonicalized.json b/freebase/data/free917.test.examples.canonicalized.json new file mode 100644 index 0000000..388cc60 --- /dev/null +++ b/freebase/data/free917.test.examples.canonicalized.json @@ -0,0 +1,278 @@ +[ + {"utterance": "how many tv programs did danny devito produce", "targetFormula": "(count (!fb:tv.tv_producer_term.program ((lambda x (fb:tv.tv_producer_term.producer (var x))) fb:en.danny_devito)))"}, + {"utterance": "how many countries are in south america", "targetFormula": "(count (fb:location.location.containedby fb:en.south_america))"}, + {"utterance": "what product lines does ipod include", "targetFormula": "(!fb:business.product_line.includes_product_lines fb:en.ipod)"}, + {"utterance": "when was starry night painted", "targetFormula": "(!fb:visual_art.artwork.date_completed fb:en.de_sterrennacht)"}, + {"utterance": "when was interstate 579 formed", "targetFormula": "(!fb:transportation.road.formed fb:en.interstate_579)"}, + {"utterance": "what area did the meiji constitution govern", "targetFormula": "(!fb:law.constitution.country fb:en.meiji_constitution)"}, + {"utterance": "who is the lyricist for spamalot", "targetFormula": "(!fb:theater.play.lyricist fb:en.spamalot)"}, + {"utterance": "what is yahoo!'s slogan", "targetFormula": "(!fb:organization.organization.slogan fb:m.019rl6)"}, + {"utterance": "what is the population of belgium", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.belgium))"}, + {"utterance": "who is the founder of savealot", "targetFormula": "(!fb:organization.organization.founders fb:en.save-a-lot)"}, + {"utterance": "what library system is the sunset branch library in", "targetFormula": "(!fb:library.public_library.library_system fb:en.sunset_branch_library)"}, + {"utterance": "where was the 3 juno asteroid discovered", "targetFormula": "(!fb:astronomy.astronomical_discovery.discovery_site fb:en.3_juno)"}, + {"utterance": "what is the population of europe", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.europe))"}, + {"utterance": "what weight class was the fight of the century", "targetFormula": "(!fb:boxing.boxing_match.weight_class fb:en.fight_of_the_century)"}, + {"utterance": "who designed pac-man", "targetFormula": "(!fb:cvg.computer_videogame.designers fb:en.pac-man)"}, + {"utterance": "when was the order of saint michael founded", "targetFormula": "(!fb:royalty.order_of_chivalry.date_founded fb:en.order_of_saint_michael)"}, + {"utterance": "what is the collection of postcards called", "targetFormula": "(!fb:interests.collection_category.name_of_collection_activity fb:en.postcard)"}, + {"utterance": "what year was ron glass an award nominee", "targetFormula": "(!fb:award.award_nomination.year ((lambda x (fb:award.award_nomination.award_nominee (var x))) fb:en.ron_glass))"}, + {"utterance": "what's the horsepower of an alluminum-alloy v6 engine", "targetFormula": "(!fb:automotive.engine.horsepower fb:en.alluminum_alloy_v6)"}, + {"utterance": "what causes prostate cancer", "targetFormula": "(!fb:medicine.disease.causes fb:en.prostate_cancer)"}, + {"utterance": "when was the iphone introduced", "targetFormula": "(!fb:computer.computer.introduced fb:en.iphone)"}, + {"utterance": "in what martial art does christopher adams have a black belt", "targetFormula": "(!fb:martial_arts.martial_arts_certification.art (and ((lambda x (fb:martial_arts.martial_arts_certification.qualification (var x))) fb:en.black_belt) ((lambda x (fb:martial_arts.martial_arts_certification.person (var x))) fb:en.christopher_adams)))"}, + {"utterance": "who was the librettist for the magic flute", "targetFormula": "(!fb:opera.opera.librettist fb:en.the_magic_flute)"}, + {"utterance": "what episode of snl did ben stiller host", "targetFormula": "(!fb:base.saturdaynightlive.snl_host.episodes_hosted fb:en.ben_stiller)"}, + {"utterance": "who created the philosopher_s stone", "targetFormula": "(!fb:fictional_universe.fictional_object.created_by fb:en.philosophers_stone)"}, + {"utterance": "how many organizations are in the automobile industry", "targetFormula": "(count (fb:business.business_operation.industry fb:en.automobile))"}, + {"utterance": "what was procter & gamble's net profit in 1955", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:business.business_operation.net_profit (var x))) fb:en.procter_gamble))"}, + {"utterance": "what is currency code for uk currency", "targetFormula": "(!fb:finance.currency.currency_code fb:en.uk)"}, + {"utterance": "what's the symbol for mercury", "targetFormula": "(!fb:chemistry.chemical_element.symbol fb:m.025sw5g)"}, + {"utterance": "what is jerry seinfeld religion", "targetFormula": "(!fb:people.person.religion fb:en.jerry_seinfeld)"}, + {"utterance": "how big is the screen on a nikon coolpix s50", "targetFormula": "(!fb:digicams.digital_camera.lcd_screen_dimensions fb:en.nikon_coolpix_s50)"}, + {"utterance": "how many speeches did winston churchill give", "targetFormula": "(count (!fb:event.speech_or_presentation.presented_work ((lambda x (fb:event.speech_or_presentation.speaker_s (var x))) fb:en.winston_churchill)))"}, + {"utterance": "how many religions use the bible", "targetFormula": "(count (!fb:religion.religious_text.religious_text_of fb:en.bible))"}, + {"utterance": "what is the capital of iceland", "targetFormula": "(!fb:location.country.capital fb:en.iceland)"}, + {"utterance": "what public transportation is there in tokyo", "targetFormula": "(!fb:travel.travel_destination.local_transportation fb:en.tokyo)"}, + {"utterance": "what is europe 's area", "targetFormula": "(!fb:location.location.area fb:en.europe)"}, + {"utterance": "how many players are in the current roster of the new york mets", "targetFormula": "(count (!fb:baseball.baseball_roster_position.player ((lambda x (fb:baseball.baseball_roster_position.team (var x))) fb:en.new_york_mets)))"}, + {"utterance": "what are the treatments of prostate cancer", "targetFormula": "(!fb:medicine.disease.treatments fb:en.prostate_cancer)"}, + {"utterance": "where was the peseta used as currency", "targetFormula": "(!fb:finance.currency.countries_formerly_used fb:en.spanish_peseta)"}, + {"utterance": "what is the currency code for the spanish peseta", "targetFormula": "(!fb:finance.currency.currency_code fb:en.spanish_peseta)"}, + {"utterance": "how many awards did big daddy win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.big_daddy)))"}, + {"utterance": "what bridges go over the san francisco bay", "targetFormula": "(!fb:geography.body_of_water.bridges fb:en.san_francisco_bay)"}, + {"utterance": "how many notable people died by poisoning", "targetFormula": "(count (fb:people.deceased_person.cause_of_death fb:en.poison))"}, + {"utterance": "what parks are in the canadian national parks system", "targetFormula": "(!fb:protected_sites.park_system.member_parks fb:en.canadian_national_parks)"}, + {"utterance": "who instructed steven seagal", "targetFormula": "(!fb:martial_arts.martial_artist.instructor fb:en.steven_seagal)"}, + {"utterance": "when did the last episode of six feet under air", "targetFormula": "(!fb:tv.tv_program.air_date_of_final_episode fb:en.six_feet_under)"}, + {"utterance": "how many major events happened in australia", "targetFormula": "(count (!fb:location.location.events fb:en.australia))"}, + {"utterance": "how many religions believe in reincarnation", "targetFormula": "(count (!fb:religion.belief.belief_of fb:en.reincarnation))"}, + {"utterance": "what other titles does 13 going on 30 have", "targetFormula": "(!fb:common.topic.alias fb:en.13_going_on_30)"}, + {"utterance": "how many writing systems are used in japanese", "targetFormula": "(count (!fb:language.human_language.writing_system fb:en.japanese))"}, + {"utterance": "who is the ceo of savealot", "targetFormula": "(!fb:organization.leadership.person (and ((lambda x (fb:organization.leadership.role (var x))) fb:en.chief_executive_officer) ((lambda x (fb:organization.leadership.organization (var x))) fb:en.save-a-lot)))"}, + {"utterance": "what starts are in ursa minor", "targetFormula": "(!fb:astronomy.constellation.contains fb:en.ursa_minor)"}, + {"utterance": "how many conferences have been held at the los angeles convention center", "targetFormula": "(count (!fb:conferences.conference_venue.conferences fb:en.los_angeles_convention_center))"}, + {"utterance": "when was diet mountain dew created", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.mountain_dew))"}, + {"utterance": "what are some mountain bike models", "targetFormula": "(!fb:bicycles.bicycle_type.bicycle_models_of_this_type fb:en.mountain_bike)"}, + {"utterance": "how many films has julie andrews been in", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.julie_edwards)))"}, + {"utterance": "who designed the costumes for alice in wonderland", "targetFormula": "(!fb:film.film.costume_design_by fb:m.04jpg2p)"}, + {"utterance": "what mountain range is king edward peak in", "targetFormula": "(!fb:geography.mountain.mountain_range fb:en.king_edward_peak)"}, + {"utterance": "what are some hotels in vancouver", "targetFormula": "(!fb:travel.travel_destination.accommodation fb:en.vancouver_british_columbia)"}, + {"utterance": "what year were the ny yankees founded", "targetFormula": "(!fb:sports.sports_team.founded fb:en.new_york_yankees)"}, + {"utterance": "how many national parks does the national wildlife refuge have", "targetFormula": "(count (!fb:protected_sites.park_system.member_parks fb:en.national_wildlife_refuge))"}, + {"utterance": "how many actors use their middle name or initial", "targetFormula": "(count (!fb:freebase.list.entries fb:m.0599h7b))"}, + {"utterance": "where is fry_s turkish delight sold", "targetFormula": "(!fb:food.candy_bar.sold_in fb:en.frys_turkish_delight)"}, + {"utterance": "who invented koolaid", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.kool-aid))"}, + {"utterance": "when was doritos produced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.doritos))"}, + {"utterance": "how many countries use the spanish peseta", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.spanish_peseta))"}, + {"utterance": "when was the pencarrow head lighthouse first lit", "targetFormula": "(!fb:architecture.lighthouse.year_first_lit fb:en.pencarrow_head_lighthouse)"}, + {"utterance": "what titles does the world boxing association have", "targetFormula": "(!fb:boxing.boxing_sanctioning_body.titles fb:en.world_boxing_association)"}, + {"utterance": "how many species does the san francisco zoo have", "targetFormula": "(!fb:zoos.zoo.num_species fb:en.san_francisco_zoo)"}, + {"utterance": "what round did thrilla in manila end", "targetFormula": "(!fb:boxing.boxing_match.round_match_ended fb:en.thrilla_in_manila)"}, + {"utterance": "what is pycon about", "targetFormula": "(!fb:conferences.conference_series.subject fb:en.pycon)"}, + {"utterance": "how many works have been lost due to theft", "targetFormula": "(count (!fb:media_common.cause_of_loss.works_lost_this_way fb:en.theft))"}, + {"utterance": "what armed forces does thailand have", "targetFormula": "(!fb:military.military_combatant.armed_forces fb:en.thailand)"}, + {"utterance": "what martial arts does chris maden practice", "targetFormula": "(!fb:martial_arts.martial_artist.martial_art fb:en.chris_maden)"}, + {"utterance": "when was koolaid invented", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.kool-aid))"}, + {"utterance": "what year did sgt. pepper's lonely hearts club band win a grammy", "targetFormula": "(!fb:award.award_honor.year ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.sgt_peppers_lonely_hearts_club_band))"}, + {"utterance": "how many islands are there in lake superior", "targetFormula": "(count (!fb:geography.body_of_water.islands fb:en.lake_superior))"}, + {"utterance": "how many countries is spanish spoken in", "targetFormula": "(count (!fb:language.human_language.countries_spoken_in fb:en.spanish))"}, + {"utterance": "when was the ss great britain established as a museum", "targetFormula": "(!fb:architecture.museum.established fb:en.ss_great_britain)"}, + {"utterance": "what type of rock is the marcellus formation", "targetFormula": "(!fb:geology.geological_formation.type_of_rock fb:en.marcellus_formation)"}, + {"utterance": "when was capri sun developed", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.capri_sun))"}, + {"utterance": "when was 300 released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.300_2007)"}, + {"utterance": "how many wins did the philadelphia eagles have in the 2008 nfl season", "targetFormula": "(count (!fb:sports.sports_team_season_record.wins (and ((lambda x (fb:sports.sports_team_season_record.season (var x))) fb:en.2008_nfl_season) ((lambda x (fb:sports.sports_team_season_record.team (var x))) fb:en.philadelphia_eagles))))"}, + {"utterance": "when was barbie launched", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.barbie))"}, + {"utterance": "how many libretti did wagner write", "targetFormula": "(count (!fb:opera.librettist.libretti fb:en.richardwagner))"}, + {"utterance": "what is bruce almighty rated", "targetFormula": "(!fb:film.film.rating fb:en.bruce_almighty)"}, + {"utterance": "where was lady washington built", "targetFormula": "(!fb:boats.ship.place_built fb:en.lady_washington)"}, + {"utterance": "how many domains are in the science & technology category", "targetFormula": "(count (!fb:freebase.domain_category.domains fb:en.science_technology))"}, + {"utterance": "what is the building function of the eiffel tower", "targetFormula": "(!fb:architecture.building.building_function fb:en.eiffel_tower)"}, + {"utterance": "how many children does danny devito have", "targetFormula": "(count (!fb:people.person.children fb:en.danny_devito))"}, + {"utterance": "how many different industries are there in home depot", "targetFormula": "(count (!fb:business.business_operation.industry fb:en.the_home_depot))"}, + {"utterance": "at what school was delta delta delta founded", "targetFormula": "(!fb:education.fraternity_sorority.founded_location fb:en.delta_delta_delta)"}, + {"utterance": "in what season of stargate sg-1 is the episode show and tell", "targetFormula": "(!fb:tv.tv_series_episode.season fb:m.08mpny)"}, + {"utterance": "who started starbucks", "targetFormula": "(!fb:organization.organization.founders fb:en.starbucks)"}, + {"utterance": "who is the captain of the edmonton oilers", "targetFormula": "(!fb:ice_hockey.hockey_team.captain fb:en.edmonton_oilers)"}, + {"utterance": "who are the founders of home depot", "targetFormula": "(!fb:organization.organization.founders fb:en.the_home_depot)"}, + {"utterance": "what are the celtic languages", "targetFormula": "(!fb:language.language_family.languages fb:en.celtic_languages)"}, + {"utterance": "what is the state flower of alaska", "targetFormula": "(!fb:location.location_symbol_relationship.symbol (and ((lambda x (fb:location.location_symbol_relationship.administrative_division (var x))) fb:en.alaska) ((lambda x (fb:location.location_symbol_relationship.Kind_of_symbol (var x))) fb:en.state_flower)))"}, + {"utterance": "what are the theme areas at disneyland", "targetFormula": "(!fb:amusement_parks.park.areas fb:en.disneyland)"}, + {"utterance": "what production is the sopranos currently in", "targetFormula": "(!fb:tv.tv_program.currently_in_production fb:en.the_sopranos)"}, + {"utterance": "where does the chow chow originate", "targetFormula": "(!fb:biology.animal_breed.place_of_origin fb:en.chow_chow)"}, + {"utterance": "what position did mike schmidt play", "targetFormula": "(!fb:baseball.baseball_player.position_s fb:en.mike_schmidt)"}, + {"utterance": "what breed group is a shar pei in the american kennel club", "targetFormula": "(!fb:biology.breed_registration.breed_group (and ((lambda x (fb:biology.breed_registration.breed (var x))) fb:en.shar_pei) ((lambda x (fb:biology.breed_registration.registry (var x))) fb:en.american_kennel_club)))"}, + {"utterance": "where was polonium discovered", "targetFormula": "(!fb:chemistry.chemical_element.discovering_country fb:en.polonium)"}, + {"utterance": "how many episodes of taylor made piano were there", "targetFormula": "(count (!fb:radio.radio_program.episodes fb:en.taylor_made_piano_a_jazz_history))"}, + {"utterance": "what record label was ali farka toure signed to", "targetFormula": "(!fb:music.artist.label fb:en.ali_farka_toure)"}, + {"utterance": "what team does richard hamilton play for", "targetFormula": "(!fb:basketball.basketball_roster_position.team ((lambda x (fb:basketball.basketball_roster_position.player (var x))) fb:en.richard_hamilton))"}, + {"utterance": "what is africa 's population", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.africa))"}, + {"utterance": "what are the isotopes of zinc", "targetFormula": "(!fb:chemistry.chemical_element.isotopes fb:en.zinc)"}, + {"utterance": "who published the amazing spider-man", "targetFormula": "(!fb:comic_books.comic_book_series.publisher fb:en.the_amazing_spider-man)"}, + {"utterance": "how thick is the aletsch glacier", "targetFormula": "(!fb:geography.glacier.thickness fb:en.aletsch_glacier)"}, + {"utterance": "when was the release date for titanic", "targetFormula": "(!fb:film.film.initial_release_date fb:en.titanic_special_edition_dvd)"}, + {"utterance": "what are some research only cancer centers", "targetFormula": "(!fb:medicine.cancer_center_type.centers_of_this_kind fb:en.research_only)"}, + {"utterance": "what is the theme song of full house", "targetFormula": "(!fb:tv.tv_program.theme_song fb:en.full_house)"}, + {"utterance": "how was pluto discovered", "targetFormula": "(!fb:astronomy.astronomical_discovery.discovery_technique fb:en.pluto)"}, + {"utterance": "what versions of mac os x is mozilla firefox compatible with", "targetFormula": "(!fb:computer.software_compatibility.operating_system (!fb:computer.software.compatible_oses fb:en.mozilla_firefox))"}, + {"utterance": "how many seasons of seinfeld are there", "targetFormula": "(count (!fb:tv.tv_program.seasons fb:en.seinfeld))"}, + {"utterance": "in 1982 who were the primetieme emmy award for comedy series nominees", "targetFormula": "(!fb:award.award_nomination.award_nominee (and ((lambda x (fb:award.award_nomination.year (var x))) (date 1982 -1 -1)) ( (lambda x (fb:award.award_nomination.award (var x))) fb:en.primetime_emmy_award_for_outstanding_comedy_series)))"}, + {"utterance": "what company owns nutter butter", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.nutter_butter))"}, + {"utterance": "when was facebook launched", "targetFormula": "(!fb:internet.website.launched fb:en.facebook)"}, + {"utterance": "what position does cristiano ronaldo play", "targetFormula": "(!fb:soccer.football_player.position_s fb:en.cristiano_ronaldo)"}, + {"utterance": "how many games has hasbro published", "targetFormula": "(count (!fb:games.game_publisher.games_published fb:en.hasbro))"}, + {"utterance": "what universe is the lord of the rings set in", "targetFormula": "(!fb:fictional_universe.work_of_fiction.setting fb:en.the_lord_of_the_rings)"}, + {"utterance": "how many teams participated in the 1979 cricket world cup", "targetFormula": "(count (!fb:cricket.cricket_tournament_event.teams fb:en.1979_cricket_world_cup))"}, + {"utterance": "what year was the album decade released", "targetFormula": "(!fb:music.album.release_date fb:m.02967)"}, + {"utterance": "how many operating systems is adobe flash compatible with", "targetFormula": "(count (!fb:computer.software_compatibility.operating_system ((lambda x (fb:computer.software_compatibility.software (var x))) fb:en.adobe_flash)))"}, + {"utterance": "what is the population estimated in the world", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.earth))"}, + {"utterance": "how many tennis events are there at the olympics", "targetFormula": "(count (!fb:olympics.olympic_sport.events fb:m.07bs0))"}, + {"utterance": "how many rna codons does glycine have", "targetFormula": "(count (!fb:biology.amino_acid.rna_codons fb:en.glycine))"}, + {"utterance": "who was charlie_s angels produced by", "targetFormula": "(!fb:film.film.produced_by fb:en.charlies_angels)"}, + {"utterance": "who completed mozart_s requiem", "targetFormula": "(!fb:media_common.completion_of_unfinished_work.finisher ((lambda x (fb:media_common.completion_of_unfinished_work.unfinished_work (var x))) fb:m.02xtrr))"}, + {"utterance": "how many teams did joe torre manage", "targetFormula": "(count (!fb:baseball.baseball_historical_managerial_position.team ((lambda x (fb:baseball.baseball_historical_managerial_position.manager (var x))) fb:en.joe_torre)))"}, + {"utterance": "how many cricket bowlers use fast bowling", "targetFormula": "(count (fb:cricket.cricket_bowler.technique fb:en.fast_bowling))"}, + {"utterance": "who designed the giant dipper", "targetFormula": "(!fb:amusement_parks.ride.designer fb:en.giant_dipper)"}, + {"utterance": "what characters are in super mario bros", "targetFormula": "(!fb:cvg.game_performance.character ((lambda x (fb:cvg.game_performance.game (var x))) fb:en.super_mario_bros))"}, + {"utterance": "what was john f kennedy 's cause of death", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.john_f_kennedy)"}, + {"utterance": "what is the genre of the skeptics' guide to the universe", "targetFormula": "(!fb:broadcast.content.genre fb:m.03z9smt)"}, + {"utterance": "how many employees does nintendo have", "targetFormula": "(count (!fb:business.employment_tenure.person (!fb:business.employer.employees fb:en.nintendo)))"}, + {"utterance": "how many types of cumulus clouds are there", "targetFormula": "(count (!fb:meteorology.cloud.varieties fb:m.0csh5))"}, + {"utterance": "how many other names is ron glass known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.ron_glass))"}, + {"utterance": "what number is ryan callahan on the new york rangers", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.number (and ((lambda x (fb:ice_hockey.hockey_roster_position.team (var x))) fb:en.new_york_rangers) ((lambda x (fb:ice_hockey.hockey_roster_position.player (var x))) fb:en.ryan_callahan)))"}, + {"utterance": "what are the christian holidays", "targetFormula": "(!fb:religion.religion.holidays fb:en.christianity)"}, + {"utterance": "who directed charlie_s angels", "targetFormula": "(!fb:film.film.directed_by fb:en.charlies_angels)"}, + {"utterance": "what was jack albertson 's cause of death", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.jack_albertson)"}, + {"utterance": "when was the printing press invented", "targetFormula": "(!fb:law.invention.date_of_invention fb:en.printing_press)"}, + {"utterance": "what are the neighborhoods in new york city", "targetFormula": "(!fb:location.place_with_neighborhoods.neighborhoods fb:en.new_york_ny)"}, + {"utterance": "how many people practice karate", "targetFormula": "(count (!fb:martial_arts.martial_art.well_known_practitioner fb:en.karate))"}, + {"utterance": "what is the area of south america", "targetFormula": "(!fb:location.location.area fb:en.south_america)"}, + {"utterance": "how many sites are on the national register of historic places", "targetFormula": "(count (!fb:protected_sites.natural_or_cultural_site_listing.listed_site ( (lambda x (fb:protected_sites.natural_or_cultural_site_listing.designation (var x))) fb:en.national_register_of_historic_places)))"}, + {"utterance": "what football team does andy reid currently coach", "targetFormula": "(!fb:american_football.football_coach.current_team_head_coached fb:en.andy_reid)"}, + {"utterance": "how many protected sites does the u.s. national park service govern", "targetFormula": "(count (!fb:protected_sites.governing_body_of_protected_sites.protected_sites_governed fb:en.national_park_service))"}, + {"utterance": "what was viacom 's revenue in 2009", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (and ((lambda x (fb:measurement_unit.dated_money_value.valid_date (var x))) (date 2009 -1 -1)) ((lambda x (!fb:business.business_operation.revenue (var x))) fb:en.viacom)))"}, + {"utterance": "what island group is jekyll island a part of", "targetFormula": "(!fb:geography.island.island_group fb:en.jekyll_island)"}, + {"utterance": "how many children does jerry seinfeld have", "targetFormula": "(count (!fb:people.person.children fb:en.jerry_seinfeld))"}, + {"utterance": "what type of organism was lucy a fossil of", "targetFormula": "(!fb:biology.fossil_specimen.organism fb:m.0gd2g7)"}, + {"utterance": "when was crystal light originally marketed", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.crystal_light))"}, + {"utterance": "how many people ride the london underground daily", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:metropolitan_transit.transit_system.daily_riders fb:en.london_underground)))"}, + {"utterance": "in what disaster was old st paul_s cathedral destroyed", "targetFormula": "(!fb:event.disaster_affected_structure.destroyed_by_disaster fb:en.old_st_pauls_cathedral)"}, + {"utterance": "what instruments did omarion play", "targetFormula": "(!fb:music.group_member.instruments_played fb:en.omarion_grandberry)"}, + {"utterance": "what are the books in the chronicles of narnia series", "targetFormula": "(fb:book.written_work.part_of_series fb:en.the_chronicles_of_narnia)"}, + {"utterance": "what characters are featured in batman: the dark knight returns", "targetFormula": "(!fb:comic_books.comic_book_series.featured_characters fb:en.batman_the_dark_knight_returns)"}, + {"utterance": "what was henry viii's royal line", "targetFormula": "(!fb:royalty.monarch.royal_line fb:en.king_henry_viii_of_england)"}, + {"utterance": "what was the cost of building the magnum xl-200", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:amusement_parks.ride.cost fb:en.magnum_xl-200))"}, + {"utterance": "how many cow's milk cheeses are there", "targetFormula": "(count (!fb:food.cheese_milk_source.cheeses fb:en.cattle))"}, + {"utterance": "did the big bang exhibit at the science museum cost money", "targetFormula": "(!fb:exhibitions.exhibition_run.admission_fee (and ((lambda x (fb:exhibitions.exhibition_run.venue (var x))) fb:en.science_museum_great_britain) ((lambda x (fb:exhibitions.exhibition_run.exhibition (var x))) fb:m.046chwc)))"}, + {"utterance": "what movie did danny devito win an award for in 1981", "targetFormula": "(!fb:award.award_honor.honored_for (and ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.danny_devito) ((lambda x (fb:award.award_honor.year (var x))) (date 1981 -1 -1))))"}, + {"utterance": "who coaches the australian cricket team", "targetFormula": "(!fb:cricket.cricket_team.coach fb:en.australian_cricket_team)"}, + {"utterance": "what are the texts of taoism", "targetFormula": "(!fb:religion.religion.texts fb:en.taoism)"}, + {"utterance": "who was 8 mile directed by", "targetFormula": "(!fb:film.film.directed_by fb:en.8_mile)"}, + {"utterance": "how many tv did jerry seinfeld have a starring role in", "targetFormula": "(count (!fb:tv.regular_tv_appearance.series ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.jerry_seinfeld)))"}, + {"utterance": "when was walmart founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.wal-mart)"}, + {"utterance": "who used to be quarterback for the green bay packers", "targetFormula": "(!fb:american_football.football_historical_roster_position.player (and ( (lambda x (fb:american_football.football_historical_roster_position.position_s (var x))) fb:en.quarterback) ( (lambda x (fb:american_football.football_historical_roster_position.team (var x))) fb:en.green_bay_packers)))"}, + {"utterance": "how many teams are in the atlantic division of the eastern conference", "targetFormula": "(count (!fb:ice_hockey.hockey_division.teams fb:en.atlantic_division))"}, + {"utterance": "how many radio programs about science are there", "targetFormula": "(count (!fb:radio.radio_subject.programs_with_this_subject fb:m.06mq7))"}, + {"utterance": "when was the latest release of microsoft word", "targetFormula": "(!fb:computer.software.latest_release_date fb:en.microsoft_word)"}, + {"utterance": "what architectural style is the brooklyn bridge", "targetFormula": "(!fb:architecture.structure.architectural_style fb:en.brooklyn_bridge)"}, + {"utterance": "how many tv channels does nbc own", "targetFormula": "(count (!fb:broadcast.tv_station_owner.tv_stations fb:en.nbc))"}, + {"utterance": "how many turbojet engines are there", "targetFormula": "(count (!fb:engineering.engine_category.engines fb:en.turbojet))"}, + {"utterance": "how many inversions does the mind eraser have", "targetFormula": "(count (!fb:amusement_parks.ride.inversions fb:en.the_mind_eraser))"}, + {"utterance": "when was 13 going on 30 released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.13_going_on_30)"}, + {"utterance": "when did john j. raskob own the empire state building", "targetFormula": "(!fb:architecture.ownership.end_date (and ((lambda x (fb:architecture.ownership.owner (var x))) fb:en.john_j_raskob) ((lambda x (fb:architecture.ownership.structure (var x))) fb:en.empire_state_building)))"}, + {"utterance": "what was the date of the first sesame street episode aired", "targetFormula": "(!fb:tv.tv_program.air_date_of_first_episode fb:en.sesame_street)"}, + {"utterance": "who created the far side", "targetFormula": "(!fb:comic_strips.comic_strip_creator_duration.creator_of_strip ((lambda x (fb:comic_strips.comic_strip_creator_duration.comic_strip (var x))) fb:en.the_far_side))"}, + {"utterance": "what conditions have symptoms including headache", "targetFormula": "(fb:medicine.disease.symptoms fb:en.severe_headache)"}, + {"utterance": "what games has macsoft games developed", "targetFormula": "(!fb:cvg.cvg_developer.games_developed fb:en.macsoft_games)"}, + {"utterance": "are lithium batteries rechargable", "targetFormula": "(!fb:engineering.battery_cell_type.rechargeable fb:en.lithium_battery)"}, + {"utterance": "how many people died of a skiing accident", "targetFormula": "(count (fb:people.deceased_person.cause_of_death fb:en.skiing_accident))"}, + {"utterance": "what decision did manny pacquiao vs. timothy bradley end with", "targetFormula": "(!fb:boxing.boxing_match.decision fb:m.0j24b7h)"}, + {"utterance": "what spirits are produced in kentucky", "targetFormula": "(!fb:distilled_spirits.spirit_producing_region.distilleries fb:en.kentucky)"}, + {"utterance": "when was home depot founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.the_home_depot)"}, + {"utterance": "when did japan end as a musical group", "targetFormula": "(!fb:music.artist.active_end fb:m.016f2t)"}, + {"utterance": "what animal does marscapone cheese come from", "targetFormula": "(!fb:food.cheese.source_of_milk fb:en.mascarpone)"}, + {"utterance": "who was titanic directed by", "targetFormula": "(!fb:film.film.directed_by fb:en.titanic_special_edition_dvd)"}, + {"utterance": "what matches have had the wbc world champion title at stake", "targetFormula": "(!fb:boxing.boxing_title.matches_with_this_title_at_stake fb:m.0cj52b7)"}, + {"utterance": "how many people played in the 2010 fifa world cup final", "targetFormula": "(count (!fb:soccer.football_player_match_participation.player ( (lambda x (fb:soccer.football_player_match_participation.match (var x))) fb:en.2010_fifa_world_cup_final)))"}, + {"utterance": "how many celebrities have abused cocaine", "targetFormula": "(count (!fb:celebrities.substance_abuse_problem.celebrity ((lambda x (fb:celebrities.substance_abuse_problem.substance (var x))) fb:en.cocaine)))"}, + {"utterance": "what are the lines of the new york city subway", "targetFormula": "(!fb:metropolitan_transit.transit_system.transit_lines fb:en.new_york_city_subway)"}, + {"utterance": "when was cap'n crunch introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.capn_crunch))"}, + {"utterance": "when was oxygen discovered", "targetFormula": "(!fb:chemistry.chemical_element.discovery_date fb:en.oxygen)"}, + {"utterance": "when did easy aces stop being produced", "targetFormula": "(!fb:broadcast.content.production_end fb:en.easy_aces)"}, + {"utterance": "what team does mike babcock coach", "targetFormula": "(!fb:ice_hockey.hockey_coach.current_team fb:en.mike_babcock)"}, + {"utterance": "how many engineers worked on the design and construction of the plymouth breakwater", "targetFormula": "(count (!fb:projects.project_participation.participant (and ( (lambda x (fb:projects.project_participation.project (var x))) fb:en.design_and_construction_of_the_plymouth_breakwater) ((lambda x (fb:projects.project_participation.role (var x))) fb:en.engineer))))"}, + {"utterance": "when was the sony nex-5 released", "targetFormula": "(!fb:digicams.digital_camera.released fb:m.0cp1pcv)"}, + {"utterance": "when was letter from america last broadcast", "targetFormula": "(!fb:radio.radio_program.final_broadcast fb:en.letter_from_america)"}, + {"utterance": "in what zoo was knut kept", "targetFormula": "(!fb:zoos.animal_captivity.zoo ((lambda x (fb:zoos.animal_captivity.animal (var x))) fb:m.02q470c))"}, + {"utterance": "what is the highest drop on stealth", "targetFormula": "(!fb:amusement_parks.ride.drop fb:m.0b6_sx)"}, + {"utterance": "how many films has tim burton produced", "targetFormula": "(count (fb:film.film.produced_by fb:en.tim_burton))"}, + {"utterance": "did jack dempsey win the long count fight", "targetFormula": "(!fb:boxing.match_boxer_relationship.winner_won ((lambda x (fb:boxing.match_boxer_relationship.boxer (var x))) fb:en.jack_dempsey))"}, + {"utterance": "who were the key designers of the macintosh", "targetFormula": "(!fb:computer.computer.key_designers fb:en.macintosh)"}, + {"utterance": "how many people have won the nobel peace prize", "targetFormula": "(count (!fb:award.award_honor.award_winner ((lambda x (fb:award.award_honor.award (var x))) fb:en.nobel_peace_prize)))"}, + {"utterance": "where was omarion born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.omarion_grandberry)"}, + {"utterance": "what tourist attractions are in rome", "targetFormula": "(!fb:travel.travel_destination.tourist_attractions fb:en.rome)"}, + {"utterance": "what are the major exports of madagascar", "targetFormula": "(!fb:location.imports_exports_by_industry.industry ((lambda x (!fb:location.statistical_region.major_exports (var x))) fb:en.madagascar))"}, + {"utterance": "when was the construction of new steubenville bridge finished", "targetFormula": "(!fb:projects.project.actual_completion_date fb:m.0jb4_0h)"}, + {"utterance": "how many other names are there for jcpenney", "targetFormula": "(count (!fb:common.topic.alias fb:en.j_c_penney))"}, + {"utterance": "what language family is afrikaans part of", "targetFormula": "(!fb:language.human_language.language_family fb:en.afrikaans)"}, + {"utterance": "what percentage of the grapes in a 1966 chateau latour grand vin are merlot", "targetFormula": "(!fb:wine.grape_variety_composition.percentage (and ((lambda x (fb:wine.grape_variety_composition.wine (var x))) fb:en.1966_chateau_latour) ((lambda x (fb:wine.grape_variety_composition.grape_variety (var x))) fb:en.merlot)))"}, + {"utterance": "how many historical events happened in south america", "targetFormula": "(count (!fb:location.location.events fb:en.south_america))"}, + {"utterance": "what meteor showers has the comet halley spawned", "targetFormula": "(!fb:astronomy.comet.meteor_shower_spawned fb:en.comet_halley)"}, + {"utterance": "when was pride and prejudice published", "targetFormula": "(!fb:book.written_work.date_of_first_publication fb:en.pride_and_prejudice)"}, + {"utterance": "who manufactured millennium force", "targetFormula": "(!fb:amusement_parks.ride.manufacturer fb:en.millennium_force)"}, + {"utterance": "what characters were on the cover of batman #1", "targetFormula": "(!fb:comic_books.comic_book_issue.characters_on_cover fb:en.batman_1)"}, + {"utterance": "what sort of weave is used to make tweed", "targetFormula": "(!fb:fashion.textile.weave fb:en.tweed)"}, + {"utterance": "when gatorade was first developed", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.gatorade))"}, + {"utterance": "on how many projects was james walker a design engineer", "targetFormula": "(count (!fb:projects.project_participation.project (and ((lambda x (fb:projects.project_participation.participant (var x))) fb:en.james_walker) ((lambda x (fb:projects.project_participation.role (var x))) fb:en.design_engineer))))"}, + {"utterance": "how many awards did the movie 8 mile win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.8_mile)))"}, + {"utterance": "how many people practice buddhism", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:religion.religion.number_of_adherents fb:en.buddhism))"}, + {"utterance": "how much did the construction of the taj mahal cost", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:projects.project.actual_cost fb:en.initial_design_and_construction_of_taj_mahal))"}, + {"utterance": "what are some object-oriented programming languages", "targetFormula": "(!fb:computer.programming_language_paradigm.languages fb:en.object-oriented_programming)"}, + {"utterance": "who were the curators for renoir in the 20th century", "targetFormula": "(!fb:exhibitions.exhibition.curators fb:en.renoir_in_the_20th_century)"}, + {"utterance": "for what country did bernard lagat play in the 2000 summer olympics", "targetFormula": "(!fb:olympics.olympic_athlete_affiliation.country (and ((lambda x (fb:olympics.olympic_athlete_affiliation.olympics (var x))) fb:en.2000_summer_olympics) ((lambda x (fb:olympics.olympic_athlete_affiliation.athlete (var x))) fb:en.bernard_lagat)))"}, + {"utterance": "what titles were at stake in the the rumble in the jungle", "targetFormula": "(!fb:boxing.boxing_match.titles_at_stake fb:en.the_rumble_in_the_jungle)"}, + {"utterance": "what games has electronic arts developed", "targetFormula": "(!fb:cvg.cvg_developer.games_developed fb:en.electronic_arts)"}, + {"utterance": "how many stores are in nittany mall", "targetFormula": "(count (!fb:business.shopping_center.store fb:en.nittany_mall))"}, + {"utterance": "when was nutty professor released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.the_nutty_professor_1996)"}, + {"utterance": "who was the editor in chief of die welt in 2000", "targetFormula": "(!fb:business.employment_tenure.person (and (and ((lambda x (fb:business.employment_tenure.company (var x))) fb:en.die_welt) ((lambda x (fb:business.employment_tenure.to (var x))) (date 2000 -1 -1))) ((lambda x (fb:business.employment_tenure.title (var x))) fb:en.editor_in_chief)))"}, + {"utterance": "what is the nutty professor rated", "targetFormula": "(!fb:film.film.rating fb:en.the_nutty_professor_1996)"}, + {"utterance": "how many film performances did jack albertson do", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.jack_albertson)))"}, + {"utterance": "what musicians have died of lung cancer", "targetFormula": "(fb:people.deceased_person.cause_of_death fb:en.lung_cancer)"}, + {"utterance": "what type of bridge is the manhattan bridge", "targetFormula": "(!fb:transportation.bridge.bridge_type fb:en.manhattan_bridge)"}, + {"utterance": "what sport did scott anderson play in the 1992 summer olympics", "targetFormula": "(!fb:olympics.olympic_athlete_affiliation.sport (and ((lambda x (fb:olympics.olympic_athlete_affiliation.athlete (var x))) fb:m.04dnjr9) ((lambda x (fb:olympics.olympic_athlete_affiliation.olympics (var x))) fb:en.1992_summer_olympics)))"}, + {"utterance": "who designed the parthenon", "targetFormula": "(!fb:architecture.structure.architect fb:en.parthenon)"}, + {"utterance": "how many monarchs are from the house of tutor", "targetFormula": "(count (!fb:royalty.royal_line.monarchs_from_this_line fb:en.tudor_dynasty))"}, + {"utterance": "what german athletes have participated in the olympics", "targetFormula": "(!fb:olympics.olympic_athlete_affiliation.athlete ((lambda x (fb:olympics.olympic_athlete_affiliation.country (var x))) fb:en.germany))"}, + {"utterance": "what teams did babe ruth play for", "targetFormula": "(!fb:baseball.baseball_historical_roster_position.team ((lambda x (fb:baseball.baseball_historical_roster_position.player (var x))) fb:en.babe_ruth))"}, + {"utterance": "in what events did ian thorpe compete in the 2004 summer olympics", "targetFormula": "(!fb:olympics.olympic_medal_honor.event (and ((lambda x (fb:olympics.olympic_medal_honor.olympics (var x))) fb:en.2004_summer_olympics) ((lambda x (fb:olympics.olympic_medal_honor.medalist (var x))) fb:en.ian_thorpe)))"}, + {"utterance": "what is the wingspan of an eclipse 500", "targetFormula": "(!fb:aviation.aircraft_model.wingspan fb:en.eclipse_500)"}, + {"utterance": "who sponsors the hoby seminars", "targetFormula": "(!fb:conferences.conference_series.sponsoring_organization fb:m.0hmwcy2)"}, + {"utterance": "who founded the order of the dragon", "targetFormula": "(!fb:royalty.order_of_chivalry.founders fb:en.order_of_the_dragon)"}, + {"utterance": "who produced sabotage by the beastie boys", "targetFormula": "(!fb:music.recording.producer fb:m.0l16j8)"}, + {"utterance": "how many runs does the thunder ridge ski area have", "targetFormula": "(count (!fb:skiing.ski_area.number_of_runs fb:en.thunder_ridge_ski_area))"}, + {"utterance": "how many people died in hurricane wilma", "targetFormula": "(!fb:meteorology.tropical_cyclone.direct_fatalities fb:en.hurricane_wilma)"}, + {"utterance": "who discovered the rings of saturn", "targetFormula": "(!fb:astronomy.astronomical_discovery.discoverer fb:en.rings_of_saturn)"}, + {"utterance": "what is the lcd screen resolution of a nikon d80", "targetFormula": "(!fb:digicams.digital_camera.lcd_pixels fb:en.nikon_d80)"}, + {"utterance": "when was the san diego zoo opened", "targetFormula": "(!fb:zoos.zoo.opened fb:en.san_diego_zoo)"}, + {"utterance": "how heavy is a panasonic lumix dmc-tz3", "targetFormula": "(!fb:digicams.digital_camera.weight fb:en.panasonic_lumix_dmc_tz3)"}, + {"utterance": "where was liam gallagher born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.liam_gallagher)"}, + {"utterance": "how many beers come a can", "targetFormula": "(count (!fb:food.beer_containment.beer ((lambda x (fb:food.beer_containment.container_type (var x))) fb:en.beverage_can)))"}, + {"utterance": "what is the subject of the atlantic monthly", "targetFormula": "(!fb:book.periodical.subjects fb:en.the_atlantic_monthly)"}, + {"utterance": "what was the american past about", "targetFormula": "(!fb:radio.radio_program.subjects fb:m.05v0tbd)"}, + {"utterance": "who destroyed the one ring", "targetFormula": "(!fb:fictional_universe.fictional_object.destroyed_by fb:en.one_ring)"}, + {"utterance": "how many students are there at the university of iceland", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:education.educational_institution.total_enrollment fb:en.university_of_iceland))"}, + {"utterance": "who founded the red cross", "targetFormula": "(!fb:organization.organization.founders fb:en.american_red_cross)"}, + {"utterance": "how many radio stations does cbs radio own", "targetFormula": "(count (!fb:broadcast.radio_station_owner.radio_stations fb:en.cbs_radio))"}, + {"utterance": "what is ashok malhotra's bowling pace", "targetFormula": "(!fb:cricket.cricket_bowler.pace fb:en.ashok_malhotra)"}, + {"utterance": "how long is wired_s gadget lab podcast", "targetFormula": "(!fb:broadcast.podcast_feed.average_media_length fb:en.wireds_gadget_lab_podcast_podcast_feed)"}, + {"utterance": "how many politicians have served in the us navy", "targetFormula": "(count (!fb:military.military_service.military_person (!fb:military.armed_force.personnel fb:en.united_states_navy)))"}, + {"utterance": "what causes bipolar disorder", "targetFormula": "(!fb:medicine.disease.causes fb:en.bipolar_disorder)"}, + {"utterance": "what causes syphilis", "targetFormula": "(!fb:medicine.disease.causes fb:en.syphillis)"}, + {"utterance": "who said that_s one small step for man, one giant leap for mankind", "targetFormula": "(!fb:media_common.quotation.author fb:m.06gj5j3)"}, + {"utterance": "what bicycle models does raleigh manufacture", "targetFormula": "(!fb:bicycles.bicycle_manufacturer.bicycle_models fb:en.raleigh_bicycle_company)"}, + {"utterance": "what play was west side story adapted from", "targetFormula": "(!fb:media_common.adaptation.adapted_from fb:en.west_side_story)"}, + {"utterance": "what team does alan butcher coach", "targetFormula": "(!fb:cricket.cricket_coach.current_team fb:en.alan_butcher)"}, + {"utterance": "how many speeches have been given about world war ii", "targetFormula": "(count (!fb:event.speech_or_presentation.presented_work ((lambda x (fb:event.speech_or_presentation.speech_topic (var x))) fb:en.world_war_ii)))"}, + {"utterance": "who is the newscaster on abc 6 news", "targetFormula": "(!fb:business.employment_tenure.person (and ((lambda x (fb:business.employment_tenure.title (var x))) fb:en.news_presenter) ((lambda x (fb:business.employment_tenure.company (var x))) fb:en.abc_news_washington_dc)))"}, + {"utterance": "who is the present newscaster on cbs evening news", "targetFormula": "(!fb:tv.tv_regular_personal_appearance.person (and ((lambda x (fb:tv.tv_regular_personal_appearance.appearance_type (var x))) fb:en.newscaster) ((lambda x (fb:tv.tv_regular_personal_appearance.program (var x))) fb:en.cbs_evening_news)))"} +] diff --git a/freebase/data/free917.train.examples.canonicalized.json b/freebase/data/free917.train.examples.canonicalized.json new file mode 100644 index 0000000..4dade1d --- /dev/null +++ b/freebase/data/free917.train.examples.canonicalized.json @@ -0,0 +1,643 @@ +[ + {"utterance": "at what institutions was marshall hall a professor", "targetFormula": "(!fb:education.academic_post.institution (and ((lambda x (fb:education.academic_post.person (var x))) fb:en.marshall_hall) ((lambda x (fb:education.academic_post.position_or_title (var x))) fb:en.professor)))"}, + {"utterance": "what fuel does an internal combustion engine use", "targetFormula": "(!fb:engineering.engine.energy_source fb:en.internal_combustion_engine)"}, + {"utterance": "how many companies are traded by the nyse", "targetFormula": "(count (!fb:business.stock_ticker_symbol.ticker_symbol (!fb:finance.stock_exchange.companies_traded fb:en.new_york_stock_exchange_inc)))"}, + {"utterance": "what is the address of the apple, inc. headquarters", "targetFormula": "(!fb:location.mailing_address.citytown ((lambda x (!fb:organization.organization.headquarters (var x))) fb:en.apple_inc))"}, + {"utterance": "what was the cover price of the x-men issue 1", "targetFormula": "(!fb:measurement_unit.money_value.amount (!fb:comic_books.comic_book_issue.cover_price fb:en.the_x_men_1))"}, + {"utterance": "how often does the washington post come out", "targetFormula": "(!fb:book.periodical_frequency.issues_per_year (!fb:book.periodical.frequency_or_issues_per_year fb:en.the_washington_post))"}, + {"utterance": "what issue of sandman is a dream of a thousand cats", "targetFormula": "(!fb:comic_books.comic_book_issue.issue_number fb:en.a_dream_of_a_thousand_cats)"}, + {"utterance": "how many people survived the sinking of the titanic", "targetFormula": "(count (!fb:event.disaster.survivors fb:en.sinking_of_the_titanic))"}, + {"utterance": "what olympics has egypt participated in", "targetFormula": "(!fb:olympics.olympic_participating_country.olympics_participated_in fb:en.egypt)"}, + {"utterance": "what celebrities have gone to the betty ford center", "targetFormula": "(!fb:celebrities.rehab.celebrity ((lambda x (fb:celebrities.rehab.rehab_facility (var x))) fb:en.betty_ford_center))"}, + {"utterance": "how old do you have to be to play monopoly", "targetFormula": "(!fb:games.game.minimum_age_years fb:en.monopoly_boardgame)"}, + {"utterance": "how many first generation particles are there", "targetFormula": "(count (!fb:physics.subatomic_particle_generation.particles fb:m.0b66f5g))"}, + {"utterance": "when was the first amendment ratified", "targetFormula": "(!fb:law.constitutional_amendment.ratification_completed_on fb:en.first_amendment_to_the_united_states_constitution)"}, + {"utterance": "how many teams participate in the uefa", "targetFormula": "(count (!fb:sports.sports_league_participation.team ((lambda x (fb:sports.sports_league_participation.league (var x))) fb:en.uefa)))"}, + {"utterance": "who are the linebackers on the chicago bears", "targetFormula": "(!fb:american_football.football_roster_position.player (and ((lambda x (fb:american_football.football_roster_position.position (var x))) fb:en.linebacker) ((lambda x (fb:american_football.football_roster_position.team (var x))) fb:en.chicago_bears)))"}, + {"utterance": "what did bertrand russell die of", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.bertrand_russell)"}, + {"utterance": "how long is the haunted mansion ride at disneyland", "targetFormula": "(!fb:amusement_parks.ride.duration fb:en.haunted_mansion)"}, + {"utterance": "what type of tea is gunpowder tea", "targetFormula": "(!fb:food.tea.tea_type fb:en.gunpowder_tea)"}, + {"utterance": "what highway system does us route 101 belong to", "targetFormula": "(!fb:transportation.road.highway_system fb:en.us_route_101)"}, + {"utterance": "what league is real madrid a member of", "targetFormula": "(!fb:soccer.football_league_participation.league ((lambda x (fb:soccer.football_league_participation.team (var x))) fb:en.real_madrid))"}, + {"utterance": "how much did a canon powershot tx1 cost when it was released", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:digicams.digital_camera.street_price fb:en.powershot_tx1))"}, + {"utterance": "what regions does the lse cover", "targetFormula": "(!fb:finance.stock_exchange.primary_regions fb:en.london_stock_exchange)"}, + {"utterance": "how many people does borders employ", "targetFormula": "(count (!fb:business.employment_tenure.person (!fb:business.employer.employees fb:en.borders_group)))"}, + {"utterance": "where did monsanto rank on the forune 500 in 2000", "targetFormula": "(!fb:award.ranking.rank (and (and ((lambda x (fb:award.ranking.year (var x))) (date 2000 -1 -1)) ((lambda x (fb:award.ranking.list (var x))) fb:en.fortune_500)) ((lambda x (fb:award.ranking.item (var x))) fb:en.monsanto)))"}, + {"utterance": "how many prosthetic makeup artists worked on 28 days later", "targetFormula": "(count (!fb:film.film_crew_gig.crewmember (and ((lambda x (fb:film.film_crew_gig.film_crew_role (var x))) fb:en.prosthetic_makeup_artist) ((lambda x (fb:film.film_crew_gig.film (var x))) fb:en.28_weeks_later))))"}, + {"utterance": "how many ships has nathanael herreshoff designed", "targetFormula": "(count (!fb:boats.ship_designer.boats_designed fb:en.nathanael_herreshoff))"}, + {"utterance": "who manages fulham f. c.", "targetFormula": "(!fb:soccer.football_team_management_tenure.manager ((lambda x (fb:soccer.football_team_management_tenure.team (var x))) fb:en.fulham_fc))"}, + {"utterance": "when was the catcher in the rye first published", "targetFormula": "(!fb:book.written_work.date_of_first_publication fb:en.the_catcher_in_the_rye)"}, + {"utterance": "how many rock formations were formed during the cretaceous period", "targetFormula": "(count (fb:geology.geological_formation.formed_during_period fb:en.cretaceous))"}, + {"utterance": "what are bmw's manufacturing plants", "targetFormula": "(!fb:automotive.company.manufacturing_plants fb:en.bmw)"}, + {"utterance": "who are some practitioners of judo", "targetFormula": "(!fb:martial_arts.martial_art.well_known_practitioner fb:en.judo)"}, + {"utterance": "what is the average temperature in sydney in august", "targetFormula": "(!fb:travel.travel_destination_monthly_climate.average_max_temp_c (and ((lambda x (fb:travel.travel_destination_monthly_climate.travel_destination (var x))) fb:en.sydney) ((lambda x (fb:travel.travel_destination_monthly_climate.month (var x))) fb:en.august)))"}, + {"utterance": "over what area does the court of chancery have jurisdiction", "targetFormula": "(!fb:law.court.jurisdiction fb:en.court_of_chancery)"}, + {"utterance": "when did mount fuji last erupt", "targetFormula": "(!fb:geography.mountain.last_eruption fb:en.mount_fuji)"}, + {"utterance": "how tall is westminster abbey", "targetFormula": "(!fb:architecture.structure.height_meters fb:en.westminster_abbey)"}, + {"utterance": "when was wells fargo founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.wells_fargo)"}, + {"utterance": "when did hey arnold! stop running", "targetFormula": "(!fb:tv.tv_program.air_date_of_final_episode fb:en.hey_arnold)"}, + {"utterance": "when did christopher higgins stop playing for the montreal canadiens", "targetFormula": "(!fb:ice_hockey.hockey_previous_roster_position.to (and ( (lambda x (fb:ice_hockey.hockey_previous_roster_position.player (var x))) fb:en.christopher_higgins) ((lambda x (fb:ice_hockey.hockey_previous_roster_position.team (var x))) fb:en.montreal_canadiens)))"}, + {"utterance": "what is the hull made of on the james craig", "targetFormula": "(!fb:boats.ship.hull_material fb:m.02zl93)"}, + {"utterance": "how many pets did john f kennedy own", "targetFormula": "(count (!fb:base.famouspets.pet_ownership.pet ((lambda x (fb:base.famouspets.pet_ownership.owner (var x))) fb:en.john_f_kennedy)))"}, + {"utterance": "how many floors does the white house have", "targetFormula": "(!fb:architecture.building.floors fb:en.white_house)"}, + {"utterance": "in which comic book issue did kitty pryde first appear", "targetFormula": "(!fb:comic_books.comic_book_character.first_appearance fb:en.kitty_pryde)"}, + {"utterance": "what are the deities of hinduism", "targetFormula": "(!fb:religion.religion.deities fb:en.hinduism)"}, + {"utterance": "who designed the iphone", "targetFormula": "(!fb:computer.computer.key_designers fb:en.iphone)"}, + {"utterance": "how many other names is jerry seinfeld known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.jerry_seinfeld))"}, + {"utterance": "what are the latitude and longitude of the eiffel tower", "targetFormula": "(!fb:location.geocode.longitude (!fb:location.location.geolocation fb:en.eiffel_tower))"}, + {"utterance": "how many visits does the magic kingdom get per year", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:amusement_parks.park.annual_visits fb:en.magic_kingdom))"}, + {"utterance": "how many film performances did beyonce do", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.beyonce)))"}, + {"utterance": "who manufactures the specialized stumpjumper", "targetFormula": "(!fb:bicycles.bicycle_model.manufacturer fb:en.stumpjumper)"}, + {"utterance": "what types of fuel does a 4 cylinder engine take", "targetFormula": "(!fb:automotive.engine.fuels_used fb:en.4_cylinder)"}, + {"utterance": "what major events in history happened in africa", "targetFormula": "(!fb:location.location.events fb:en.africa)"}, + {"utterance": "what is the transmission of a 2011 honda fit", "targetFormula": "(!fb:automotive.trim_level.transmission fb:m.0h2krn6)"}, + {"utterance": "what is angelina jolie's net worth", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:celebrities.celebrity.net_worth fb:en.angelina_jolie))"}, + {"utterance": "what conferences does google sponsor", "targetFormula": "(!fb:conferences.conference_sponsor.conferences fb:en.google)"}, + {"utterance": "what is the orbital period of the moon", "targetFormula": "(!fb:astronomy.orbital_relationship.orbital_period fb:en.moon)"}, + {"utterance": "where did barosaurus live", "targetFormula": "(fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.barosaurus)"}, + {"utterance": "who wrote a series of unfortunate events", "targetFormula": "(fb:book.author.series_written_or_contributed_to fb:en.a_series_of_unfortunate_events)"}, + {"utterance": "what genre is doonesbury", "targetFormula": "(!fb:comic_strips.comic_strip.genre fb:en.doonesbury)"}, + {"utterance": "what religious order belongs to kirkstall abbey", "targetFormula": "(!fb:religion.monastery.religious_order fb:en.kirkstall_abbey)"}, + {"utterance": "during what time period did uluru form", "targetFormula": "(!fb:geology.geological_formation.formed_during_period fb:en.uluru)"}, + {"utterance": "what blogs are in german", "targetFormula": "(fb:internet.blog.language fb:en.german_language)"}, + {"utterance": "what type of clothing are knickerbockers", "targetFormula": "(!fb:fashion.garment.specialization_of fb:en.knickerbockers)"}, + {"utterance": "what is the address of the mitchell public library", "targetFormula": "(!fb:location.mailing_address.street_address ((lambda x (!fb:library.public_library.address (var x))) fb:m.0j9by57))"}, + {"utterance": "what genres did 8 mile consist of", "targetFormula": "(!fb:film.film.genre fb:en.8_mile)"}, + {"utterance": "how many types of dinosaurs were there in africa", "targetFormula": "(count (!fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.africa))"}, + {"utterance": "who is the head coach of the chicago bulls", "targetFormula": "(!fb:basketball.basketball_team.head_coach fb:en.chicago_bulls)"}, + {"utterance": "who did nathan fillion voice in halo: reach", "targetFormula": "(!fb:cvg.game_performance.character (and (and ((lambda x (fb:cvg.game_performance.game (var x))) fb:en.halo_reach) ((lambda x (fb:cvg.game_performance.performance_type (var x))) fb:m.02nsjvf)) ((lambda x (fb:cvg.game_performance.voice_actor (var x))) fb:en.nathan_fillion)))"}, + {"utterance": "what is the motto of the order of the golden fleece", "targetFormula": "(!fb:royalty.order_of_chivalry.motto fb:en.order_of_the_golden_fleece)"}, + {"utterance": "what is the retail floor space of the dubai marina mall", "targetFormula": "(!fb:business.shopping_center.retail_floor_space_m_2 fb:m.06zl3_x)"}, + {"utterance": "who built the uss nautilus", "targetFormula": "(!fb:boats.ship.ship_builder fb:en.uss_nautilus)"}, + {"utterance": "where did nathan smith die", "targetFormula": "(!fb:people.deceased_person.place_of_death fb:en.nathan_smith_1770)"}, + {"utterance": "who owns flickr", "targetFormula": "(!fb:internet.website.owner fb:en.flickr)"}, + {"utterance": "who designed the game of life", "targetFormula": "(!fb:games.game.designer fb:en.the_game_of_life)"}, + {"utterance": "what medal did graeme brown receive at the 2004 summer olympics", "targetFormula": "(!fb:olympics.olympic_medal_honor.medal (and ((lambda x (fb:olympics.olympic_medal_honor.olympics (var x))) fb:en.2004_summer_olympics) ((lambda x (fb:olympics.olympic_medal_honor.medalist (var x))) fb:en.graeme_brown)))"}, + {"utterance": "what is the symbol for yen", "targetFormula": "(!fb:finance.currency.prefix_symbol fb:en.japanese_yen)"}, + {"utterance": "what topics are equivalent to us county", "targetFormula": "(!fb:freebase.type_profile.equivalent_topic fb:location.us_county)"}, + {"utterance": "what is the origin of the mississippi river", "targetFormula": "(!fb:geography.river.origin fb:en.mississippi_river)"}, + {"utterance": "how many celebrities have been charged with driving under the influence", "targetFormula": "(count (!fb:celebrities.legal_entanglement.celebrity ((lambda x (fb:celebrities.legal_entanglement.offense (var x))) fb:en.driving_under_the_influence)))"}, + {"utterance": "how many expansion packs does the sims have", "targetFormula": "(count (!fb:cvg.computer_videogame.expansions fb:en.the_sims))"}, + {"utterance": "what is ron glass 's religion", "targetFormula": "(!fb:people.person.religion fb:en.ron_glass)"}, + {"utterance": "what is the cape may lighthouse made of", "targetFormula": "(!fb:architecture.lighthouse.construction fb:en.cape_may_lighthouse)"}, + {"utterance": "what is the subject of the source report", "targetFormula": "(!fb:radio.radio_program.subjects fb:en.the_source_report)"}, + {"utterance": "where was jerry seinfeld born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.jerry_seinfeld)"}, + {"utterance": "what was roe v. wade about", "targetFormula": "(!fb:law.legal_case.subject fb:en.roe_v_wade)"}, + {"utterance": "what position does craig adams play", "targetFormula": "(!fb:ice_hockey.hockey_player.hockey_position fb:en.craig_adams_1977)"}, + {"utterance": "how many record labels was omarion under", "targetFormula": "(count (!fb:music.artist.label fb:en.omarion_grandberry))"}, + {"utterance": "in 1981 what award did danny devito win", "targetFormula": "(!fb:award.award_honor.award (and ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.danny_devito) ((lambda x (fb:award.award_honor.year (var x))) (date 1981 -1 -1))))"}, + {"utterance": "what sizes does pilsner urquell come in", "targetFormula": "(!fb:food.beer_containment.size ((lambda x (fb:food.beer_containment.beer (var x))) fb:en.pilsner_urquell))"}, + {"utterance": "how many starring tv roles did jack albertson have", "targetFormula": "(count (!fb:tv.regular_tv_appearance.character ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.jack_albertson)))"}, + {"utterance": "when was the airspeed oxford first flown", "targetFormula": "(!fb:aviation.aircraft_model.maiden_flight fb:en.airspeed_oxford)"}, + {"utterance": "what is the release date for 2 fast 2 furious", "targetFormula": "(!fb:film.film.initial_release_date fb:en.2_fast_2_furious)"}, + {"utterance": "who founded nivea", "targetFormula": "(!fb:organization.organization.founders fb:en.nivea)"}, + {"utterance": "what are the file formats supported by the iphone", "targetFormula": "(!fb:computer.computing_platform.file_formats_supported fb:en.iphone)"}, + {"utterance": "where is the tempelhof international airport located", "targetFormula": "(!fb:location.location.containedby fb:en.tempelhof_international_airport)"}, + {"utterance": "what languages has microsoft designed", "targetFormula": "(!fb:computer.programming_language_designer.languages_designed fb:en.microsoft_corporation)"}, + {"utterance": "what works did anthony payne finish", "targetFormula": "(!fb:media_common.completion_of_unfinished_work.unfinished_work ((lambda x (fb:media_common.completion_of_unfinished_work.finisher (var x))) fb:en.anthony_payne))"}, + {"utterance": "how many awards did titanic win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.titanic_special_edition_dvd)))"}, + {"utterance": "how is the plastiki propelled", "targetFormula": "(!fb:boats.ship.means_of_propulsion fb:en.plastiki)"}, + {"utterance": "where was the 1st world science fiction convention held", "targetFormula": "(!fb:conferences.conference.venue fb:en.1st_world_science_fiction_convention)"}, + {"utterance": "what is serge made out of", "targetFormula": "(!fb:fashion.textile.fiber fb:en.serge)"}, + {"utterance": "who hosted the 2003 cricket world cup", "targetFormula": "(!fb:cricket.cricket_tournament_event.host fb:en.2003_cricket_world_cup)"}, + {"utterance": "when did secretariat die", "targetFormula": "(!fb:biology.deceased_organism.date_of_death fb:en.secretariat)"}, + {"utterance": "when was the galileo spacecraft launched", "targetFormula": "(!fb:spaceflight.satellite.launch_date fb:en.galileo_spacecraft)"}, + {"utterance": "when was savealot founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.save-a-lot)"}, + {"utterance": "how many people can play settlers of catan", "targetFormula": "(!fb:measurement_unit.integer_range.high_value (!fb:games.game.number_of_players fb:en.settlers_of_catan))"}, + {"utterance": "how many exhibitions have there been about leonardo da vinci", "targetFormula": "(count (!fb:exhibitions.exhibition_subject.exhibitions_created_about_this_subject fb:en.leonardo_da_vinci))"}, + {"utterance": "what buildings were destroyed in september 11th", "targetFormula": "(!fb:event.disaster.structures_destroyed fb:en.september_11_2001_attacks)"}, + {"utterance": "how many dinosaur species are from africa", "targetFormula": "(count (!fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.africa))"}, + {"utterance": "when was the abbey of st. gall listed as an unesco world heritage site", "targetFormula": "(!fb:protected_sites.natural_or_cultural_site_listing.date_listed (and ( (lambda x (fb:protected_sites.natural_or_cultural_site_listing.designation (var x))) fb:en.world_heritage_site) ( (lambda x (fb:protected_sites.natural_or_cultural_site_listing.listed_site (var x))) fb:en.abbey_of_st_gall)))"}, + {"utterance": "what is the capacity of a rolls royce merlin", "targetFormula": "(!fb:engineering.piston_engine.capacity fb:en.rolls-royce_merlin)"}, + {"utterance": "what are the side effects of morphine", "targetFormula": "(!fb:medicine.medical_treatment.side_effects fb:en.morphine)"}, + {"utterance": "what is the boiling point of water", "targetFormula": "(!fb:chemistry.chemical_compound.boiling_point fb:en.cpd_c00001_h2o)"}, + {"utterance": "who is the screenplay for the nutty professor by", "targetFormula": "(!fb:film.film.written_by fb:en.the_nutty_professor_1996)"}, + {"utterance": "what is big daddy rated", "targetFormula": "(!fb:film.film.rating fb:en.big_daddy)"}, + {"utterance": "when did michael jackson go solo", "targetFormula": "(!fb:music.artist.active_start fb:en.michael_jackson)"}, + {"utterance": "when did tutankhamun die", "targetFormula": "(!fb:people.deceased_person.date_of_death fb:en.tutankhamun)"}, + {"utterance": "when was the united nations founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.united_nations)"}, + {"utterance": "who are the distributors for bruce almighty", "targetFormula": "(!fb:film.film_film_distributor_relationship.distributor ((lambda x (fb:film.film_film_distributor_relationship.film (var x))) fb:en.bruce_almighty))"}, + {"utterance": "when was bruce almighty released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.bruce_almighty)"}, + {"utterance": "what was the budget for edward scissorhands", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:film.film.estimated_budget (var x))) fb:en.edward_scissorhands))"}, + {"utterance": "who owns wxpn", "targetFormula": "(!fb:broadcast.radio_station.owner fb:en.wxpn)"}, + {"utterance": "what was walter gropius' architectural style", "targetFormula": "(!fb:architecture.architect.architectural_style fb:en.walter_gropius)"}, + {"utterance": "how many beds does mclean hospital have", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:medicine.hospital.beds fb:en.mclean_hospital)))"}, + {"utterance": "what are some gato class submarines", "targetFormula": "(!fb:boats.ship_class.ships_in_class fb:en.gato_class_submarine)"}, + {"utterance": "what is the maximum altitude of stratus clouds", "targetFormula": "(!fb:meteorology.cloud.maximum_altitude_m fb:en.stratus)"}, + {"utterance": "how many conferences about mathematics are there", "targetFormula": "(count (!fb:conferences.conference_subject.series_of_conferences_about_this fb:en.mathematics))"}, + {"utterance": "what diet did ceratopsia have", "targetFormula": "(!fb:base.dinosaur.dinosaur.diet fb:en.ceratopsia)"}, + {"utterance": "what judge presided over state v. kelly", "targetFormula": "(!fb:law.legal_case.judges fb:en.state_v_kelly)"}, + {"utterance": "when was sigmod founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.sigmod)"}, + {"utterance": "in what city is h&r block headquartered", "targetFormula": "(!fb:location.mailing_address.citytown ((lambda x (!fb:organization.organization.headquarters (var x))) fb:en.h_r_block))"}, + {"utterance": "what is the highest elevation at the elk mountain ski area", "targetFormula": "(!fb:skiing.ski_area.top_elevation fb:en.elk_mountain_ski_area)"}, + {"utterance": "what is the texture of brie cheese", "targetFormula": "(!fb:food.cheese.texture fb:m.01482f)"}, + {"utterance": "how many schools have a beaver as their mascot", "targetFormula": "(count (!fb:education.school_mascot.school fb:en.beaver))"}, + {"utterance": "who created sesame street", "targetFormula": "(!fb:tv.tv_program.program_creator fb:en.sesame_street)"}, + {"utterance": "how many film performances does jerry seinfeld have", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.jerry_seinfeld)))"}, + {"utterance": "how many spin offs does sesame street have", "targetFormula": "(count (!fb:tv.tv_program.spin_offs fb:en.sesame_street))"}, + {"utterance": "what diseases cause leukemia", "targetFormula": "(!fb:medicine.disease.causes fb:en.leukemia)"}, + {"utterance": "how many different products does gsusa produce", "targetFormula": "(count (!fb:business.brand.products fb:en.girl_scouts_of_the_usa))"}, + {"utterance": "who directed meet the parents", "targetFormula": "(!fb:film.film.directed_by fb:en.meet_the_parents)"}, + {"utterance": "when was omarion born", "targetFormula": "(!fb:people.person.date_of_birth fb:en.omarion_grandberry)"}, + {"utterance": "what are some french restaurants", "targetFormula": "(!fb:dining.cuisine.restaurant fb:m.02z3r)"}, + {"utterance": "who publishes the journal of abnormal psychology", "targetFormula": "(!fb:book.periodical_publisher_period.publisher ( (lambda x (fb:book.periodical_publisher_period.periodical (var x))) fb:en.journal_of_abnormal_psychology))"}, + {"utterance": "what was sesame street 's original network", "targetFormula": "(!fb:tv.tv_network_duration.network ((lambda x (fb:tv.tv_network_duration.program (var x))) fb:en.sesame_street))"}, + {"utterance": "what are the moons of jupiter", "targetFormula": "(!fb:astronomy.orbital_relationship.orbited_by fb:en.jupiter)"}, + {"utterance": "what exhibits are there at the arignar anna zoological park", "targetFormula": "(!fb:zoos.zoo.exhibits fb:en.arignar_anna_zoological_park)"}, + {"utterance": "where was neanderthal 1 found", "targetFormula": "(!fb:biology.fossil_specimen.found_at_site fb:m.026lz95)"}, + {"utterance": "what radio program is a right to death an episode of", "targetFormula": "(!fb:radio.radio_program_episode.program fb:en.a_right_to_death)"}, + {"utterance": "what category of martial arts is tai chi chuan", "targetFormula": "(!fb:martial_arts.martial_art.category fb:en.tai_chi_chuan)"}, + {"utterance": "how many religions worship god", "targetFormula": "(count (!fb:religion.deity.deity_of fb:en.god))"}, + {"utterance": "what are the subdisciplines of engineering", "targetFormula": "(!fb:education.field_of_study.subdisciplines fb:en.engineering)"}, + {"utterance": "what is target 's industry", "targetFormula": "(!fb:business.business_operation.industry fb:en.target_corporation)"}, + {"utterance": "how many countries use the australian dollar", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.australian_dollar))"}, + {"utterance": "what software has claris developed", "targetFormula": "(!fb:computer.software_developer.software fb:en.claris)"}, + {"utterance": "what bus operators run buses to paris", "targetFormula": "(!fb:travel.transportation.transport_operator ((lambda x (fb:travel.transportation.travel_destination (var x))) fb:en.paris))"}, + {"utterance": "when was porgy and bess first performed", "targetFormula": "(!fb:opera.opera.date_of_first_performance fb:en.porgy_and_bess)"}, + {"utterance": "how many people have been influenced by socrates", "targetFormula": "(count (!fb:influence.influence_node.influenced fb:en.socrates))"}, + {"utterance": "how many runways does the san francisco international airport have", "targetFormula": "(!fb:aviation.airport.number_of_runways fb:en.san_francisco_international_airport)"}, + {"utterance": "what color did adolf anderssen play in the immortal game", "targetFormula": "(!fb:chess.chess_game_participation.color (and ((lambda x (fb:chess.chess_game_participation.game (var x))) fb:en.immortal_game) ((lambda x (fb:chess.chess_game_participation.player (var x))) fb:en.adolf_anderssen)))"}, + {"utterance": "when was the boston herald first published", "targetFormula": "(!fb:book.periodical_publication_date.date (!fb:book.periodical.first_issue_date fb:en.boston_herald))"}, + {"utterance": "is the university of lima public or private", "targetFormula": "(!fb:education.educational_institution.school_type fb:en.university_of_lima)"}, + {"utterance": "what exhibitions has the gross clinic been in", "targetFormula": "(!fb:exhibitions.exhibit.exhibitions_displayed_in fb:en.the_gross_clinic)"}, + {"utterance": "what do cirrus clouds look like", "targetFormula": "(!fb:meteorology.cloud.appearance fb:m.0cs96)"}, + {"utterance": "who played in the game of the century", "targetFormula": "(!fb:chess.chess_game_participation.player ((lambda x (fb:chess.chess_game_participation.game (var x))) fb:en.the_game_of_the_century))"}, + {"utterance": "how many countries participated in the 2006 winter olympics", "targetFormula": "(!fb:olympics.olympic_games.number_of_countries fb:en.2006_winter_olympics)"}, + {"utterance": "how many tv shows did danny devito have a leading role in", "targetFormula": "(count (!fb:tv.regular_tv_appearance.character ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.danny_devito)))"}, + {"utterance": "what awards has the english patient won", "targetFormula": "(!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.the_english_patient))"}, + {"utterance": "who designed the first generation ford mustang", "targetFormula": "(!fb:automotive.generation.designer fb:en.first_generation_ford_mustang)"}, + {"utterance": "where is the hobbit set", "targetFormula": "(!fb:fictional_universe.work_of_fiction.setting fb:en.the_hobbit)"}, + {"utterance": "who discovered neon", "targetFormula": "(!fb:chemistry.chemical_element.discoverer fb:en.neon)"}, + {"utterance": "what are some british locomotive class 52 trains", "targetFormula": "(!fb:rail.locomotive_class.locomotives_of_this_class fb:en.british_rail_class_52)"}, + {"utterance": "what number was david akers on the philadelphia eagles", "targetFormula": "(!fb:american_football.football_historical_roster_position.number (and ( (lambda x (fb:american_football.football_historical_roster_position.team (var x))) fb:en.philadelphia_eagles) ( (lambda x (fb:american_football.football_historical_roster_position.player (var x))) fb:en.david_akers)))"}, + {"utterance": "how many shows was hayden panettiere on", "targetFormula": "(count (!fb:tv.regular_tv_appearance.character ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.hayden_panettiere)))"}, + {"utterance": "how many free exhibits have there been at the science museum", "targetFormula": "(count (!fb:exhibitions.exhibition_run.exhibition ((lambda x (fb:exhibitions.exhibition_run.venue (var x))) fb:en.science_museum_great_britain)))"}, + {"utterance": "how did samuel beckett die", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.samuel_beckett)"}, + {"utterance": "what type of infection is syphilis", "targetFormula": "(!fb:medicine.infectious_disease.infectious_agent_type fb:en.syphillis)"}, + {"utterance": "what is africa 's area", "targetFormula": "(!fb:location.location.area fb:en.africa)"}, + {"utterance": "how long are adventure time episodes", "targetFormula": "(!fb:tv.tv_program.episode_running_time fb:m.0280hsm)"}, + {"utterance": "who designed the hawker sea hawk", "targetFormula": "(!fb:aviation.aircraft_model.designed_by fb:en.hawker_sea_hawk)"}, + {"utterance": "when was the high court of australia founded", "targetFormula": "(!fb:law.court.founded fb:en.high_court_of_australia)"}, + {"utterance": "who brews pauwel kwak", "targetFormula": "(!fb:food.beer.brewery_brand fb:en.pauwel_kwak)"}, + {"utterance": "when did everything is illuminated win the guardian first book award", "targetFormula": "(!fb:award.award_honor.year (and ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.everything_is_illuminated) ((lambda x (fb:award.award_honor.award (var x))) fb:en.guardian_first_book_award)))"}, + {"utterance": "what airlines is the london heathrow airport a hub for", "targetFormula": "(!fb:aviation.airport.hub_for fb:en.london_heathrow_airport)"}, + {"utterance": "who are some notable agnostic figures", "targetFormula": "(!fb:religion.religion.notable_figures fb:en.agnosticism)"}, + {"utterance": "what lines does the london overground operate", "targetFormula": "(!fb:rail.railway_operator_relationship.railway ((lambda x (fb:rail.railway_operator_relationship.operator (var x))) fb:en.london_overground))"}, + {"utterance": "how many judges has the high court of justice had", "targetFormula": "(count (!fb:law.judicial_tenure.judge ((lambda x (fb:law.judicial_tenure.court (var x))) fb:en.high_court_of_justice)))"}, + {"utterance": "what other names is the united nations known by", "targetFormula": "(!fb:common.topic.alias fb:en.united_nations)"}, + {"utterance": "where did taekwondo originate", "targetFormula": "(!fb:martial_arts.martial_art.origin fb:en.taekwondo)"}, + {"utterance": "what is the area of north america", "targetFormula": "(!fb:location.location.area fb:en.north_america)"}, + {"utterance": "since when has the arabic alphabet been used", "targetFormula": "(!fb:language.language_writing_system.used_from fb:en.arabic_alphabet)"}, + {"utterance": "how long is the mexican federal highway 1", "targetFormula": "(!fb:transportation.road.length fb:en.mexican_federal_highway_1)"}, + {"utterance": "how many schools have a chapter of sigma phi epsilon", "targetFormula": "(count (!fb:education.fraternity_sorority.colleges_and_universities fb:en.sigma_phi_epsilon))"}, + {"utterance": "who were the astronauts in the apollo 12 mission", "targetFormula": "(!fb:spaceflight.space_mission.astronauts fb:en.apollo_12)"}, + {"utterance": "how many people influenced karl marx", "targetFormula": "(count (!fb:influence.influence_node.influenced_by fb:en.karl_marx))"}, + {"utterance": "how many canon digital cameras are there", "targetFormula": "(count (!fb:digicams.digital_camera_manufacturer.cameras fb:en.canon))"}, + {"utterance": "what is the original language of the wind-up bird chronicle", "targetFormula": "(!fb:book.written_work.original_language fb:en.the_wind-up_bird_chronicle)"}, + {"utterance": "what is the usa money currency code", "targetFormula": "(!fb:finance.currency.currency_code fb:en.us)"}, + {"utterance": "how many military units originated in delaware", "targetFormula": "(count (!fb:military.military_unit_place_of_origin.military_units fb:en.delaware))"}, + {"utterance": "how many people have held the title of grand master of the order of the most holy annunciation", "targetFormula": "(count (!fb:royalty.chivalric_order_position_tenure.officer (and ( (lambda x (fb:royalty.chivalric_order_position_tenure.chivalric_office (var x))) fb:en.grand_master) ( (lambda x (fb:royalty.chivalric_order_position_tenure.order (var x))) fb:en.order_of_the_most_holy_annunciation))))"}, + {"utterance": "how many malls does oxford properties own", "targetFormula": "(count (!fb:business.shopping_center_owner.shopping_centers_owned fb:en.oxford_properties))"}, + {"utterance": "what qualification does woody strode have in seishindo kenpo", "targetFormula": "(!fb:martial_arts.martial_arts_certification.qualification (and ((lambda x (fb:martial_arts.martial_arts_certification.art (var x))) fb:en.seishindo_kenpo) ((lambda x (fb:martial_arts.martial_arts_certification.person (var x))) fb:en.woody_strode)))"}, + {"utterance": "what is the single letter abreviation for tryptophan", "targetFormula": "(!fb:biology.amino_acid.single_letter_abbreviation fb:en.tryptophan)"}, + {"utterance": "where are dravidian languages spoken", "targetFormula": "(!fb:language.language_family.geographic_distribution fb:en.dravidian_languages)"}, + {"utterance": "how many books does the detroit public library have", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:library.public_library_system.collection_size fb:en.detroit_public_library)))"}, + {"utterance": "how many kinds of footwear are there", "targetFormula": "(count (!fb:fashion.garment.more_specialized_forms fb:en.footwear))"}, + {"utterance": "in what park is splash mountain", "targetFormula": "(fb:amusement_parks.park.rides fb:en.splash_mountain)"}, + {"utterance": "what is danny devito 's religion", "targetFormula": "(!fb:people.person.religion fb:en.danny_devito)"}, + {"utterance": "what powers does sonic the hedgehog have", "targetFormula": "(!fb:fictional_universe.fictional_character.powers_or_abilities fb:en.sonic_the_hedgehog)"}, + {"utterance": "how many wins has larry bird had in his career", "targetFormula": "(!fb:basketball.basketball_coach.season_wins fb:en.larry_bird)"}, + {"utterance": "what are the symptoms of motor neuron disease", "targetFormula": "(!fb:medicine.disease.symptoms fb:en.motor_neuron_disease)"}, + {"utterance": "what was the final air date of lux radio theater", "targetFormula": "(!fb:radio.radio_program.final_broadcast fb:en.lux_radio_theater)"}, + {"utterance": "how many grammy awards did beyonce receive in her singing career", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.beyonce)))"}, + {"utterance": "what language is lohengrin in", "targetFormula": "(!fb:opera.opera.language fb:m.09hvx)"}, + {"utterance": "which issues of x-men have iceman on the cover", "targetFormula": "(!fb:comic_books.comic_book_character.cover_appearances fb:en.iceman)"}, + {"utterance": "what number is kevin youkilis on the boston red sox", "targetFormula": "(!fb:sports.sports_team_roster.number (and ((lambda x (fb:sports.sports_team_roster.player (var x))) fb:en.kevin_youkilis) ((lambda x (fb:sports.sports_team_roster.team (var x))) fb:en.boston_red_sox)))"}, + {"utterance": "what are the symptoms of prostate cancer", "targetFormula": "(!fb:medicine.disease.symptoms fb:en.prostate_cancer)"}, + {"utterance": "who created charlie brown", "targetFormula": "(!fb:fictional_universe.fictional_character.character_created_by fb:en.charlie_brown)"}, + {"utterance": "what class of train was heavy harry", "targetFormula": "(!fb:rail.locomotive.locomotive_class fb:m.0glzm0h)"}, + {"utterance": "what type of bicycle is a bianchi pista", "targetFormula": "(!fb:bicycles.bicycle_model.bicycle_type fb:en.bianchi_pista)"}, + {"utterance": "how many broadcasts does blogtalkradio distribute", "targetFormula": "(count (!fb:broadcast.distributor.distributes fb:en.blogtalkradio))"}, + {"utterance": "what label does julien fournie design for", "targetFormula": "(!fb:fashion.designer_label_association.label ((lambda x (fb:fashion.designer_label_association.designer (var x))) fb:en.julien_fournie))"}, + {"utterance": "how high is niagara falls", "targetFormula": "(!fb:geography.waterfall.height fb:en.niagara_falls)"}, + {"utterance": "who currently plays for the san jose sharks", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.player ((lambda x (fb:ice_hockey.hockey_roster_position.team (var x))) fb:en.san_jose_sharks))"}, + {"utterance": "what are the survival rates for prostate cancer", "targetFormula": "(!fb:medicine.survival_rate.rate ((lambda x (!fb:medicine.disease.survival_rates (var x))) fb:en.prostate_cancer))"}, + {"utterance": "how many award nominations did danny devito have", "targetFormula": "(count (!fb:award.award_nomination.award ((lambda x (fb:award.award_nomination.award_nominee (var x))) fb:en.danny_devito)))"}, + {"utterance": "when did charlie sheen enter the thousand oaks rehab facility", "targetFormula": "(!fb:celebrities.rehab.entered (and ((lambda x (fb:celebrities.rehab.rehab_facility (var x))) fb:m.04fz39z) ((lambda x (fb:celebrities.rehab.celebrity (var x))) fb:en.charlie_sheen)))"}, + {"utterance": "what bowling technique does matthew hayden use", "targetFormula": "(!fb:cricket.cricket_bowler.technique fb:en.matthew_hayden)"}, + {"utterance": "what's the focus of the last minute blog", "targetFormula": "(!fb:internet.blog.focus fb:en.the_last_minute_blog)"}, + {"utterance": "how many nicknames does shaquille o'neal have", "targetFormula": "(count (!fb:common.topic.alias fb:en.shaquille_oneal))"}, + {"utterance": "in what division do the st. louis blues play", "targetFormula": "(!fb:ice_hockey.hockey_team.division fb:m.06x6s)"}, + {"utterance": "who managed liverpool f.c. from 2004 to june 2010", "targetFormula": "(!fb:soccer.football_team_management_tenure.manager (and ((lambda x (fb:soccer.football_team_management_tenure.to (var x))) (date 2004 -1 -1)) ((lambda x (fb:soccer.football_team_management_tenure.team (var x))) fb:en.liverpool_fc)))"}, + {"utterance": "what is the height restriction on the roller soaker", "targetFormula": "(!fb:amusement_parks.ride.height_restriction fb:en.roller_soaker)"}, + {"utterance": "what is currency code for the canadian dollar", "targetFormula": "(!fb:finance.currency.currency_code fb:en.canadian_dollar)"}, + {"utterance": "what is the area of antarctica", "targetFormula": "(!fb:location.location.area fb:en.antarctica)"}, + {"utterance": "how many games did donovan mcnabb play in the 2008 season", "targetFormula": "(!fb:american_football.player_game_statistics.games (and ((lambda x (fb:american_football.player_game_statistics.season (var x))) fb:en.2008_nfl_season) ((lambda x (fb:american_football.player_game_statistics.player (var x))) fb:en.donovan_mcnabb)))"}, + {"utterance": "how many issues are there in the x-men", "targetFormula": "(!fb:comic_books.comic_book_series.number_of_issues fb:m.02hqp38)"}, + {"utterance": "in how many countries is the euro used", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.euro))"}, + {"utterance": "what was target 's rank in the fortune 500 in 2010", "targetFormula": "(!fb:award.ranking.rank (and (and ((lambda x (fb:award.ranking.list (var x))) fb:en.fortune_500) ((lambda x (fb:award.ranking.year (var x))) (date 2010 -1 -1))) ((lambda x (fb:award.ranking.item (var x))) fb:en.target_corporation)))"}, + {"utterance": "how many people were at the 2006 fifa world cup final", "targetFormula": "(count (!fb:soccer.football_match.attendance fb:en.2006_fifa_world_cup_final))"}, + {"utterance": "how many awards did danny devito win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.danny_devito)))"}, + {"utterance": "how many islands are in the antilles", "targetFormula": "(count (!fb:geography.island_group.islands_in_group fb:en.antilles))"}, + {"utterance": "what is the closet city to snowshoe mountain", "targetFormula": "(!fb:skiing.ski_area.closest_city fb:en.snowshoe_mountain)"}, + {"utterance": "how tall is jerry seinfeld", "targetFormula": "(!fb:people.person.height_meters fb:en.jerry_seinfeld)"}, + {"utterance": "how many cards do you need to play canasta", "targetFormula": "(!fb:games.playing_card_game.number_of_cards fb:en.canasta)"}, + {"utterance": "how many games are in the final fantasy series", "targetFormula": "(count (!fb:cvg.game_series.games_in_series fb:en.final_fantasy))"}, + {"utterance": "what are nicknames for new york", "targetFormula": "(!fb:common.topic.alias fb:en.new_york_state)"}, + {"utterance": "what is the highway fuel economy of a 2007 honda civic hybrid", "targetFormula": "(!fb:automotive.us_fuel_economy.highway_mpg ((lambda x (fb:automotive.us_fuel_economy.trim_level (var x))) fb:m.04nbbrp))"}, + {"utterance": "who is the developer of java language", "targetFormula": "(!fb:computer.programming_language.developers fb:m.07sbkfb)"}, + {"utterance": "how many speeds does a panasonic dx3000 have", "targetFormula": "(!fb:bicycles.bicycle_model.speeds fb:en.panasonic_dx3000)"}, + {"utterance": "when was joni mitchell inducted into the rock and roll hall of fame", "targetFormula": "(!fb:award.hall_of_fame_induction.date (and ((lambda x (fb:award.hall_of_fame_induction.inductee (var x))) fb:en.joni_mitchell) ((lambda x (fb:award.hall_of_fame_induction.hall_of_fame (var x))) fb:en.rock_and_roll_hall_of_fame)))"}, + {"utterance": "where was william shakespeare born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.william_shakespeare)"}, + {"utterance": "what are the dimensions of the matrix reloaded image", "targetFormula": "(!fb:common.image.size fb:m.0kjl0q)"}, + {"utterance": "how many test conferences has rudi koertzen refereed", "targetFormula": "(!fb:cricket.cricket_umpire.test_matches_refereed fb:en.rudi_koertzen)"}, + {"utterance": "what religions branched from roman catholicism", "targetFormula": "(!fb:religion.religion.branched_into fb:en.roman_catholicism)"}, + {"utterance": "what area does kifm serve", "targetFormula": "(!fb:broadcast.radio_station.serves_area fb:en.kifm)"}, + {"utterance": "who developed the polio vaccine", "targetFormula": "(!fb:medicine.vaccine.developed_by fb:en.polio_vaccine)"}, + {"utterance": "who painted the mona lisa", "targetFormula": "(!fb:visual_art.artwork.artist fb:en.mona_lisa)"}, + {"utterance": "how many collections did barry halper have", "targetFormula": "(count (!fb:interests.collection.category ((lambda x (fb:interests.collection.collector (var x))) fb:en.barry_halper)))"}, + {"utterance": "who writes the dabble blog", "targetFormula": "(!fb:internet.blog.blogger fb:en.the_dabble_blog)"}, + {"utterance": "what conventions has hillary clinton spoken at", "targetFormula": "(!fb:base.politicalconventions.convention_speech.venue ( (lambda x (fb:base.politicalconventions.convention_speech.speaker (var x))) fb:en.hillary_rodham_clinton))"}, + {"utterance": "who are some bauhaus architects", "targetFormula": "(!fb:architecture.architectural_style.architects fb:en.international_style)"}, + {"utterance": "how many teams has curly lambeau coached", "targetFormula": "(count (!fb:american_football.football_historical_coach_position.team ( (lambda x (fb:american_football.football_historical_coach_position.coach (var x))) fb:en.curly_lambeau)))"}, + {"utterance": "how many branches does the vancouver public library have", "targetFormula": "(count (!fb:library.public_library_system.branches fb:en.vancouver_public_library))"}, + {"utterance": "what city is the ben may main library in", "targetFormula": "(!fb:location.mailing_address.citytown ((lambda x (!fb:library.public_library.address (var x))) fb:m.02ncllz))"}, + {"utterance": "what guidebooks are there for san francisco", "targetFormula": "(!fb:travel.travel_destination.guidebooks fb:en.san_francisco)"}, + {"utterance": "what are some psychology journals", "targetFormula": "(!fb:education.field_of_study.journals_in_this_discipline fb:en.psychology)"}, + {"utterance": "where was the kepler spacecraft launched", "targetFormula": "(!fb:spaceflight.satellite.launch_site fb:en.kepler_mission)"}, + {"utterance": "how many film actors are there on freebase", "targetFormula": "(!fb:freebase.type_profile.instance_count fb:film.actor)"}, + {"utterance": "when did a prairie home companion first air", "targetFormula": "(!fb:radio.radio_program.first_broadcast fb:en.a_prairie_home_companion)"}, + {"utterance": "what genres does meet the parents consist of", "targetFormula": "(!fb:film.film.genre fb:en.meet_the_parents)"}, + {"utterance": "what is the max speed of a gloster meteor", "targetFormula": "(!fb:aviation.aircraft_model.maximum_speed_km_h fb:en.gloster_meteor)"}, + {"utterance": "what position does colby armstrong play on the toronto maple leafs", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.position (and ((lambda x (fb:ice_hockey.hockey_roster_position.player (var x))) fb:en.colby_armstrong) ((lambda x (fb:ice_hockey.hockey_roster_position.team (var x))) fb:en.toronto_maple_leafs)))"}, + {"utterance": "how many countries are in the european union", "targetFormula": "(count (!fb:organization.organization_membership.member ((lambda x (fb:organization.organization_membership.organization (var x))) fb:en.european_union)))"}, + {"utterance": "when was civilization first released", "targetFormula": "(!fb:cvg.computer_videogame.release_date fb:m.01vvf)"}, + {"utterance": "on what network was gilligan_s island first aired", "targetFormula": "(!fb:tv.tv_network_duration.network ((lambda x (fb:tv.tv_network_duration.program (var x))) fb:en.gilligans_island))"}, + {"utterance": "what series did tamora pierce write", "targetFormula": "(!fb:book.author.series_written_or_contributed_to fb:en.tamora_pierce)"}, + {"utterance": "who presented the gettysburg address", "targetFormula": "(!fb:event.speech_or_presentation.speaker_s ((lambda x (fb:event.speech_or_presentation.presented_work (var x))) fb:en.gettysburg_address))"}, + {"utterance": "what is the mascot of the boston red sox", "targetFormula": "(!fb:sports.sports_team.team_mascot fb:en.boston_red_sox)"}, + {"utterance": "when was cathy landers certified as a fifth degree black belt in seishindo kenpo", "targetFormula": "(!fb:martial_arts.martial_arts_certification.date (and (and ((lambda x (fb:martial_arts.martial_arts_certification.qualification (var x))) fb:en.fifth_degree) ((lambda x (fb:martial_arts.martial_arts_certification.art (var x))) fb:en.seishindo_kenpo)) ((lambda x (fb:martial_arts.martial_arts_certification.person (var x))) fb:en.cathy_landers)))"}, + {"utterance": "how many locations are there on freebase", "targetFormula": "(!fb:freebase.type_profile.instance_count fb:location.location)"}, + {"utterance": "who are gatorade 's major sponsors", "targetFormula": "(!fb:business.sponsorship.sponsored_recipient ((lambda x (fb:business.sponsorship.sponsored_by (var x))) fb:en.gatorade))"}, + {"utterance": "what is the fuel economy of a 2008 hyundai accent gls sedan in the city", "targetFormula": "(!fb:automotive.us_fuel_economy.highway_mpg ((lambda x (fb:automotive.us_fuel_economy.trim_level (var x))) fb:m.04nbpg5))"}, + {"utterance": "where is between the lines produced", "targetFormula": "(!fb:broadcast.content.location fb:m.03d5yt5)"}, + {"utterance": "when was the oreo cookie introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.oreo))"}, + {"utterance": "what positon does ray lewis currently play for the baltimore ravens", "targetFormula": "(!fb:american_football.football_historical_roster_position.position_s (and ( (lambda x (fb:american_football.football_historical_roster_position.player (var x))) fb:en.ray_lewis) ( (lambda x (fb:american_football.football_historical_roster_position.team (var x))) fb:en.baltimore_ravens)))"}, + {"utterance": "to whom is invisible man dedicated", "targetFormula": "(!fb:media_common.dedication.dedicated_to ((lambda x (fb:media_common.dedication.work_dedicated (var x))) fb:en.invisible_man))"}, + {"utterance": "how is syphilis transmitted", "targetFormula": "(!fb:medicine.infectious_disease.transmission fb:en.syphillis)"}, + {"utterance": "what characters sing soprano in tristan and isolde", "targetFormula": "(!fb:opera.opera_character_voice.character (and ((lambda x (fb:opera.opera_character_voice.opera (var x))) fb:en.tristan_und_isolde) ((lambda x (fb:opera.opera_character_voice.voice (var x))) fb:en.crystal_clear_soprano)))"}, + {"utterance": "what team did rick carlisle coach in 2002", "targetFormula": "(!fb:basketball.basketball_historical_coach_position.team (and ((lambda x (fb:basketball.basketball_historical_coach_position.from (var x))) (date 2002 -1 -1)) ((lambda x (fb:basketball.basketball_historical_coach_position.coach (var x))) fb:en.rick_carlisle)))"}, + {"utterance": "how many dialects of arabic are there", "targetFormula": "(count (!fb:language.human_language.dialects fb:en.arabic_language))"}, + {"utterance": "when did invertigo open", "targetFormula": "(!fb:amusement_parks.ride.opened fb:m.0flmt0)"}, + {"utterance": "what is the melting point of gold", "targetFormula": "(!fb:chemistry.chemical_element.melting_point fb:m.025rs2z)"}, + {"utterance": "what is the thinker made of", "targetFormula": "(!fb:visual_art.artwork.media fb:en.the_thinker)"}, + {"utterance": "who did jun matsumoto play in hana yori dango", "targetFormula": "(!fb:tv.regular_tv_appearance.character (and ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.jun_matsumoto) ((lambda x (fb:tv.regular_tv_appearance.series (var x))) fb:m.05f52tw)))"}, + {"utterance": "how many cars are in jay leno's car collection", "targetFormula": "(count (!fb:interests.collection.items (and ((lambda x (fb:interests.collection.collector (var x))) fb:en.jay_leno) ((lambda x (fb:interests.collection.category (var x))) fb:en.automobile))))"}, + {"utterance": "when did coronet peak open", "targetFormula": "(!fb:skiing.ski_area.opening_date fb:en.coronet_peak)"}, + {"utterance": "what series was game of thrones adapted from", "targetFormula": "(!fb:media_common.adaptation.adapted_from fb:en.game_of_thrones)"}, + {"utterance": "what was the science of survival exhibit about", "targetFormula": "(!fb:exhibitions.exhibition.subjects fb:m.046chvq)"}, + {"utterance": "how many legal offences has lindsey lohan committed", "targetFormula": "(count (!fb:celebrities.legal_entanglement.location ((lambda x (fb:celebrities.legal_entanglement.celebrity (var x))) fb:en.lindsay_lohan)))"}, + {"utterance": "how many people visit the new york public library annually", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:library.public_library_system.annual_visits fb:en.new_york_public_library)))"}, + {"utterance": "how many comic stories has adrienne roy colored", "targetFormula": "(count (!fb:comic_books.comic_book_colorist.comic_stories_colored fb:en.adrienne_roy))"}, + {"utterance": "what games has garry kasparov won", "targetFormula": "(!fb:chess.chess_game_participation.game ((lambda x (fb:chess.chess_game_participation.outcome (var x))) fb:m.0454vkd))"}, + {"utterance": "how tall do you have to be to ride the american eagle", "targetFormula": "(!fb:amusement_parks.ride.height_restriction fb:m.0cdlng)"}, + {"utterance": "\ufeffwho is the head coach of the pittsburgh steelers", "targetFormula": "(!fb:american_football.football_team.current_head_coach fb:en.pittsburgh_steelers)"}, + {"utterance": "what conditions does aspirin treat", "targetFormula": "(!fb:medicine.medical_treatment.used_to_treat fb:en.aspirin)"}, + {"utterance": "for what team does fedor tyutin play", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.team ((lambda x (fb:ice_hockey.hockey_roster_position.player (var x))) fb:en.fedor_tyutin))"}, + {"utterance": "how many tv shows was ron glass in", "targetFormula": "(count (!fb:tv.regular_tv_appearance.series ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.ron_glass)))"}, + {"utterance": "how many ships were built by bath iron works", "targetFormula": "(count (!fb:boats.ship_builder.ships_built fb:en.bath_iron_works))"}, + {"utterance": "who was nominated for the academy award for best director in 2011", "targetFormula": "(!fb:award.award_nomination.award_nominee (and ((lambda x (fb:award.award_nomination.award (var x))) fb:en.academy_award_for_best_director) ((lambda x (fb:award.award_nomination.year (var x))) (date 2011 -1 -1))))"}, + {"utterance": "what character says \"i'll be back\"", "targetFormula": "(!fb:media_common.quotation.spoken_by_character fb:m.06zh9s)"}, + {"utterance": "what is britney spears' sexual orientation", "targetFormula": "(!fb:celebrities.sexual_orientation_phase.sexual_orientation ((lambda x (fb:celebrities.sexual_orientation_phase.celebrity (var x))) fb:en.britney_spears))"}, + {"utterance": "how many engines have a straight-4 piston configuration", "targetFormula": "(count (!fb:engineering.piston_configuration.engines fb:en.straight-4))"}, + {"utterance": "what players scored in the 2006 fifa world cup final", "targetFormula": "(!fb:soccer.football_goal.scorer ((lambda x (fb:soccer.football_goal.match (var x))) fb:en.2006_fifa_world_cup_final))"}, + {"utterance": "how many athletes have gotten an olympic gold medal", "targetFormula": "(count (!fb:olympics.olympic_medal_honor.medalist ((lambda x (fb:olympics.olympic_medal_honor.medal (var x))) fb:en.gold_medal)))"}, + {"utterance": "what movies were nominated for favorite comedy movie", "targetFormula": "(!fb:award.award_nomination.nominated_for ( (lambda x (fb:award.award_nomination.award (var x))) fb:en.peoples_choice_award_for_favorite_comedy_movie))"}, + {"utterance": "who lived in fallingwater", "targetFormula": "(!fb:architecture.occupancy.occupant ((lambda x (fb:architecture.occupancy.building (var x))) fb:en.fallingwater))"}, + {"utterance": "who developed capri sun", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.capri_sun))"}, + {"utterance": "what lodges are in the alta ski area", "targetFormula": "(!fb:skiing.ski_area.lodges fb:en.alta_ski_area)"}, + {"utterance": "does the canon powershot a75 have an orientation sensor", "targetFormula": "(!fb:digicams.digital_camera.orientation_sensor fb:en.canon_powershot_a75)"}, + {"utterance": "how long is titanic 's runtime", "targetFormula": "(!fb:film.film_cut.runtime ((lambda x (fb:film.film_cut.film (var x))) fb:en.titanic_special_edition_dvd))"}, + {"utterance": "what team does mike brown coach", "targetFormula": "(!fb:basketball.basketball_coach.team fb:en.mike_brown_1970)"}, + {"utterance": "what team does joe girardi currently manage", "targetFormula": "(!fb:baseball.baseball_manager.current_team_managed fb:en.joe_girardi)"}, + {"utterance": "when was country time produced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.country_time))"}, + {"utterance": "who is the ceo of apple", "targetFormula": "(!fb:organization.leadership.person (and ((lambda x (fb:organization.leadership.role (var x))) fb:en.chief_executive_officer) ((lambda x (fb:organization.leadership.organization (var x))) fb:en.apple_inc)))"}, + {"utterance": "how many commanders were there in the civil war", "targetFormula": "(count (!fb:military.military_command.military_commander ((lambda x (fb:military.military_command.military_conflict (var x))) fb:en.american_civil_war)))"}, + {"utterance": "when did francesco sabatini start working on the puerta de san vicente", "targetFormula": "(!fb:projects.project_participation.from_date (and ((lambda x (fb:projects.project_participation.project (var x))) fb:m.0gk9x46) ((lambda x (fb:projects.project_participation.participant (var x))) fb:en.francesco_sabatini)))"}, + {"utterance": "when did martin margiela start designing for herm", "targetFormula": "(!fb:fashion.designer_label_association.from_date (and ((lambda x (fb:fashion.designer_label_association.designer (var x))) fb:en.martin_margiela) ((lambda x (fb:fashion.designer_label_association.label (var x))) fb:m.06dr8q)))"}, + {"utterance": "who directed 2 fast 2 furious", "targetFormula": "(!fb:film.film.directed_by fb:en.2_fast_2_furious)"}, + {"utterance": "how many languages has jrr tolkein created", "targetFormula": "(count (!fb:language.language_creator.languages_created fb:en.j_r_r_tolkien))"}, + {"utterance": "what telescopes does the lowell observatory have", "targetFormula": "(!fb:astronomy.astronomical_observatory.telescope_s fb:en.lowell_observatory)"}, + {"utterance": "how long is the tokyu tamagawa line", "targetFormula": "(!fb:rail.railway.length fb:en.tokyu_tamagawa_line)"}, + {"utterance": "who was the newscaster in 1948 on cbs evening news", "targetFormula": "(!fb:tv.tv_regular_personal_appearance.person (and (and ((lambda x (fb:tv.tv_regular_personal_appearance.appearance_type (var x))) fb:en.newscaster) ((lambda x (fb:tv.tv_regular_personal_appearance.program (var x))) fb:en.cbs_evening_news)) ((lambda x (fb:tv.tv_regular_personal_appearance.from (var x))) (date 1948 -1 -1))))"}, + {"utterance": "who is the music by in bruce almighty", "targetFormula": "(!fb:film.film.music fb:en.bruce_almighty)"}, + {"utterance": "how many episodes does sesame street have", "targetFormula": "(count (!fb:tv.tv_program.episodes fb:en.sesame_street))"}, + {"utterance": "what is the area of australia", "targetFormula": "(!fb:location.location.area fb:en.australia)"}, + {"utterance": "what is the chemical formula for butane", "targetFormula": "(!fb:chemistry.chemical_compound.formula fb:en.butane)"}, + {"utterance": "how many seasons of sesame street are there", "targetFormula": "(count (!fb:tv.tv_program.seasons fb:en.sesame_street))"}, + {"utterance": "who created superman", "targetFormula": "(!fb:comic_books.comic_book_character.created_by fb:en.superman)"}, + {"utterance": "what is the runtime for meet the parents", "targetFormula": "(!fb:film.film_cut.runtime ((lambda x (fb:film.film_cut.film (var x))) fb:en.meet_the_parents))"}, + {"utterance": "who was the casting director for meet the parents", "targetFormula": "(!fb:film.film.film_casting_director fb:en.meet_the_parents)"}, + {"utterance": "how many generations of the ipod are there", "targetFormula": "(count (!fb:business.consumer_product.product_line fb:en.ipod))"}, + {"utterance": "how many awards did jack albertson win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.jack_albertson)))"}, + {"utterance": "since when has the california gull been the state bird of utah", "targetFormula": "(!fb:location.location_symbol_relationship.date_adopted (and (and ((lambda x (fb:location.location_symbol_relationship.administrative_division (var x))) fb:en.utah) ((lambda x (fb:location.location_symbol_relationship.Kind_of_symbol (var x))) fb:en.state_bird)) ((lambda x (fb:location.location_symbol_relationship.symbol (var x))) fb:en.california_gull)))"}, + {"utterance": "what are the new york giants also known as", "targetFormula": "(!fb:common.topic.alias fb:en.new_york_giants)"}, + {"utterance": "when was universal studios japan opened", "targetFormula": "(!fb:amusement_parks.park.opened fb:en.universal_studios_japan)"}, + {"utterance": "how many people worked on the design and construction of the golden gate bridge", "targetFormula": "(count (!fb:projects.project_participation.participant ( (lambda x (fb:projects.project_participation.project (var x))) fb:en.design_and_construction_of_the_golden_gate_bridge)))"}, + {"utterance": "does doom have a multiplayer mode", "targetFormula": "(and fb:en.doom (fb:cvg.computer_videogame.gameplay_modes fb:en.multiplayer_game))"}, + {"utterance": "when was starbucks founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.starbucks)"}, + {"utterance": "how much money in damages did the 1904 toronto fire cost", "targetFormula": "(!fb:measurement_unit.money_value.amount (!fb:event.disaster.damage fb:en.1904_toronto_fire))"}, + {"utterance": "what newspapers does carnegie mellon publish", "targetFormula": "(!fb:education.educational_institution.newspaper fb:en.carnegie_mellon_university)"}, + {"utterance": "how many schools are in the school district of philadelphia", "targetFormula": "(count (!fb:education.school_district.schools fb:en.school_district_of_philadelphia))"}, + {"utterance": "what rehab facilities has jerry garcia gone to", "targetFormula": "(!fb:celebrities.rehab.rehab_facility ((lambda x (fb:celebrities.rehab.celebrity (var x))) fb:en.jerry_garcia))"}, + {"utterance": "how many peabody award winners are there", "targetFormula": "(count (!fb:award.award_honor.award_winner ((lambda x (fb:award.award_honor.award (var x))) fb:en.peabody_award)))"}, + {"utterance": "how many games are there for nintendo ds", "targetFormula": "(count (!fb:cvg.cvg_platform.games_on_this_platform fb:en.nintendo_ds))"}, + {"utterance": "who invented scrabble", "targetFormula": "(!fb:law.invention.inventor fb:en.scrabble)"}, + {"utterance": "where was the armory show exhibited", "targetFormula": "(!fb:exhibitions.exhibition_run.venue ((lambda x (fb:exhibitions.exhibition_run.exhibition (var x))) fb:en.armory_show))"}, + {"utterance": "what stadium do the phillies play in", "targetFormula": "(!fb:sports.sports_team.arena_stadium fb:en.philadelphia_phillies)"}, + {"utterance": "who was the director of the nutty professor", "targetFormula": "(!fb:film.film.directed_by fb:en.the_nutty_professor_1996)"}, + {"utterance": "what is the spin of a positron", "targetFormula": "(!fb:physics.particle.spin fb:en.positron)"}, + {"utterance": "how many brands does sara lee own", "targetFormula": "(count (!fb:business.company_brand_relationship.brand ((lambda x (fb:business.company_brand_relationship.company (var x))) fb:en.sara_lee)))"}, + {"utterance": "who was the 22nd president", "targetFormula": "(fb:government.us_president.presidency_number (number 22.0 fb:en.unitless))"}, + {"utterance": "what genre is the hound of the baskervilles", "targetFormula": "(fb:media_common.literary_genre.books_in_this_genre fb:en.the_hound_of_the_baskervilles)"}, + {"utterance": "what is the dry mass of the hubble space telescope", "targetFormula": "(!fb:spaceflight.satellite.dry_mass_kg fb:en.hubble_space_telescope)"}, + {"utterance": "what are some examples of a road bike", "targetFormula": "(!fb:bicycles.bicycle_type.bicycle_models_of_this_type fb:en.road_bicycle)"}, + {"utterance": "what particles are in the lepton family", "targetFormula": "(!fb:physics.particle_family.particles fb:en.lepton)"}, + {"utterance": "what category does the film domain belong to", "targetFormula": "(!fb:freebase.domain_profile.category fb:m.010s)"}, + {"utterance": "what rock is gruta das torres made of", "targetFormula": "(!fb:geology.geological_formation.type_of_rock fb:en.gruta_das_torres)"}, + {"utterance": "what processor did the apple ii use", "targetFormula": "(!fb:computer.computer.processor fb:en.apple_ii)"}, + {"utterance": "when is jerry seinfeld 's birthday", "targetFormula": "(!fb:people.person.date_of_birth fb:en.jerry_seinfeld)"}, + {"utterance": "what language is tagliablog in", "targetFormula": "(!fb:internet.blog.language fb:en.tagliablog)"}, + {"utterance": "who produced the three doctors", "targetFormula": "(!fb:tv.tv_producer_episode_credit.producer (!fb:tv.tv_series_episode.producers fb:en.the_three_doctors))"}, + {"utterance": "who manufactured the space shuttle discovery", "targetFormula": "(!fb:spaceflight.spacecraft.manufacturer fb:en.space_shuttle_discovery)"}, + {"utterance": "what is the population of asia", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.asia))"}, + {"utterance": "how many areas were affected by hurricane ivan", "targetFormula": "(count (!fb:meteorology.tropical_cyclone.affected_areas fb:en.hurricane_ivan))"}, + {"utterance": "when was kid_s corner first broadcast", "targetFormula": "(!fb:radio.radio_program.first_broadcast fb:m.05v47f_)"}, + {"utterance": "what is currency code for the mexican peso", "targetFormula": "(!fb:finance.currency.currency_code fb:en.mexican_peso)"}, + {"utterance": "what languages use bengali script", "targetFormula": "(!fb:language.language_writing_system.languages fb:en.bengali_script)"}, + {"utterance": "how tall is mount everest", "targetFormula": "(!fb:geography.mountain.elevation fb:en.mount_everest)"}, + {"utterance": "who are the hosts of car talk", "targetFormula": "(!fb:broadcast.content.artist fb:en.car_talk)"}, + {"utterance": "where was henry iii buried", "targetFormula": "(!fb:people.deceased_person.place_of_burial fb:en.henry_iii_of_england)"}, + {"utterance": "who manufactures the t-38 talon", "targetFormula": "(!fb:aviation.aircraft_model.manufacturer fb:en.t-38_talon)"}, + {"utterance": "what award does the royal institute of british architects present", "targetFormula": "(!fb:award.award_presenting_organization.awards_presented fb:en.royal_institute_of_british_architects)"}, + {"utterance": "how many orders did queen victoria found", "targetFormula": "(count (!fb:royalty.chivalric_order_founder.orders_founded fb:en.victoria_of_the_united_kingdom))"}, + {"utterance": "what was robert oppenheimer's role in the manhattan project", "targetFormula": "(!fb:projects.project_participation.role (and ((lambda x (fb:projects.project_participation.project (var x))) fb:en.manhattan_project) ((lambda x (fb:projects.project_participation.participant (var x))) fb:en.robert_oppenheimer)))"}, + {"utterance": "who are the players currently on the los angeles dodgers", "targetFormula": "(!fb:baseball.baseball_roster_position.player ((lambda x (fb:baseball.baseball_roster_position.team (var x))) fb:en.los_angeles_dodgers))"}, + {"utterance": "what religion did martin luther found", "targetFormula": "(!fb:religion.founding_figure.religion_founded fb:en.martin_luther)"}, + {"utterance": "what character was born in gotham city", "targetFormula": "(!fb:fictional_universe.fictional_setting.fictional_characters_born_here fb:en.gotham_city)"}, + {"utterance": "how much protein is in quinoa", "targetFormula": "(!fb:food.nutrition_fact.quantity (and ((lambda x (fb:food.nutrition_fact.food (var x))) fb:en.quinoa) ((lambda x (fb:food.nutrition_fact.nutrient (var x))) fb:en.protein)))"}, + {"utterance": "when was the movie big daddy released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.big_daddy)"}, + {"utterance": "what city hosted the 1948 summer olympics", "targetFormula": "(!fb:olympics.olympic_games.host_city fb:en.1948_summer_olympics)"}, + {"utterance": "how many beers are there on freebase", "targetFormula": "(!fb:freebase.type_profile.instance_count fb:food.beer)"}, + {"utterance": "when was target founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.target_corporation)"}, + {"utterance": "how many countries use uk money", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.uk))"}, + {"utterance": "what are columbia university's colors", "targetFormula": "(!fb:education.educational_institution.colors fb:en.columbia_university)"}, + {"utterance": "what instrument did lightnin' hopkins play", "targetFormula": "(!fb:music.group_member.instruments_played fb:en.lightnin_hopkins)"}, + {"utterance": "what are the practices of sikhism", "targetFormula": "(!fb:religion.religion.practices fb:en.sikhism)"}, + {"utterance": "how many people blog on beyond robson", "targetFormula": "(count (!fb:internet.blog.blogger fb:en.beyond_robson))"}, + {"utterance": "how many industries does walmart consist of", "targetFormula": "(count (!fb:business.business_operation.industry fb:en.wal-mart))"}, + {"utterance": "what is the population in iowa", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.iowa))"}, + {"utterance": "where does the market-frankford line stop", "targetFormula": "(!fb:metropolitan_transit.transit_line.stops fb:en.market-frankford_line)"}, + {"utterance": "what are the subclasses of the boson class", "targetFormula": "(!fb:physics.particle_family.subclasses fb:en.boson)"}, + {"utterance": "how long is the golden gate bridge", "targetFormula": "(!fb:transportation.bridge.total_length fb:en.golden_gate_bridge)"}, + {"utterance": "how many boats does the royal navy own", "targetFormula": "(count (!fb:boats.ship_ownership.ship ((lambda x (fb:boats.ship_ownership.owner (var x))) fb:en.navy_dept)))"}, + {"utterance": "where was luke skywalker born", "targetFormula": "(!fb:fictional_universe.fictional_character.place_of_birth fb:en.luke_skywalker)"}, + {"utterance": "who was the story of charlie_s angels by", "targetFormula": "(!fb:film.film.written_by fb:en.charlies_angels)"}, + {"utterance": "what was the manhattan project about", "targetFormula": "(!fb:projects.project.project_focus fb:en.manhattan_project)"}, + {"utterance": "what party did grover cleveland belong to", "targetFormula": "(!fb:government.political_party_tenure.party ((lambda x (fb:government.political_party_tenure.politician (var x))) fb:en.grover_cleveland))"}, + {"utterance": "what is the passenger capacity of a cessna citation x", "targetFormula": "(!fb:measurement_unit.integer_range.high_value (!fb:aviation.aircraft_model.passengers fb:en.cessna_citation_x))"}, + {"utterance": "when was panasonic corporation founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.matsushita_electric_industrial_co)"}, + {"utterance": "who got the gold medal in men_s singles tennis at the 1896 summer olympics", "targetFormula": "(!fb:olympics.olympic_medal_honor.medalist (and ( (lambda x (fb:olympics.olympic_medal_honor.event (var x))) fb:en.tennis_at_the_1896_summer_olympics_mens_singles) ((lambda x (fb:olympics.olympic_medal_honor.medal (var x))) fb:en.gold_medal)))"}, + {"utterance": "in what year did harry potter and the goblet of fire win the hugo award for best novel", "targetFormula": "(!fb:award.award_honor.year (and ((lambda x (fb:award.award_honor.award (var x))) fb:en.hugo_award_for_best_novel) ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.harry_potter_and_the_goblet_of_fire)))"}, + {"utterance": "how many dinosaur species are there in north america", "targetFormula": "(count (!fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.north_america))"}, + {"utterance": "what is the parent disease of a stroke", "targetFormula": "(!fb:medicine.disease.parent_disease fb:en.stroke)"}, + {"utterance": "who directed 13 going on 30", "targetFormula": "(!fb:film.film.directed_by fb:en.13_going_on_30)"}, + {"utterance": "what team does dennis bergkamp play for", "targetFormula": "(!fb:soccer.football_roster_position.team ((lambda x (fb:soccer.football_roster_position.player (var x))) fb:en.dennis_bergkamp))"}, + {"utterance": "in how many military conflicts did mussolini command", "targetFormula": "(count (!fb:military.military_command.military_conflict ((lambda x (fb:military.military_command.military_commander (var x))) fb:en.benito_mussolini)))"}, + {"utterance": "what structures did frank lloyd wright design", "targetFormula": "(!fb:architecture.architect.structures_designed fb:en.frank_lloyd_wright)"}, + {"utterance": "in what time zone is illinois", "targetFormula": "(!fb:location.location.time_zones fb:en.illinois)"}, + {"utterance": "who are the main cast members of sesame street", "targetFormula": "(!fb:tv.regular_tv_appearance.actor ((lambda x (fb:tv.regular_tv_appearance.series (var x))) fb:en.sesame_street))"}, + {"utterance": "who are the characters in to kill a mockingbird", "targetFormula": "(!fb:book.book.characters fb:en.to_kill_a_mockingbird)"}, + {"utterance": "who are the students of bruce lee", "targetFormula": "(!fb:martial_arts.martial_artist.martial_arts_students fb:en.bruce_lee)"}, + {"utterance": "how tall was seabiscuit", "targetFormula": "(!fb:biology.organism.height_meters fb:en.seabiscuit)"}, + {"utterance": "who won muhammad ali vs. joe frazier ii", "targetFormula": "(!fb:boxing.match_boxer_relationship.boxer (and ((lambda x (fb:boxing.match_boxer_relationship.winner_won (var x))) true) ((lambda x (fb:boxing.match_boxer_relationship.match (var x))) fb:en.ali-frazier_ii)))"}, + {"utterance": "what is the engine in a 2010 ferrari california", "targetFormula": "(!fb:automotive.trim_level.engine fb:m.0h33xbs)"}, + {"utterance": "how many other names is panasonic corporation known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.matsushita_electric_industrial_co))"}, + {"utterance": "what genre did lil hardin armstrong represent", "targetFormula": "(!fb:music.artist.genre fb:en.lil_hardin_armstrong)"}, + {"utterance": "what is currency code for the japanese yen", "targetFormula": "(!fb:finance.currency.currency_code fb:en.japanese_yen)"}, + {"utterance": "what was the fawlty towers episode after the psychiatrist", "targetFormula": "(!fb:tv.tv_series_episode.next_episode fb:en.the_psychiatrist)"}, + {"utterance": "who is the president of gap, inc.", "targetFormula": "(!fb:business.employment_tenure.person (and ((lambda x (fb:business.employment_tenure.company (var x))) fb:en.gap_inc) ((lambda x (fb:business.employment_tenure.title (var x))) fb:en.president)))"}, + {"utterance": "what chemical series is tungsten in", "targetFormula": "(!fb:chemistry.chemical_element.chemical_series fb:en.tungsten)"}, + {"utterance": "how many works have been set on the moon", "targetFormula": "(count (!fb:fictional_universe.fictional_setting.works_set_here fb:en.moon))"}, + {"utterance": "when was jcpenney founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.j_c_penney)"}, + {"utterance": "what is the setting of the magician_s nephew", "targetFormula": "(!fb:fictional_universe.work_of_fiction.setting fb:en.the_magicians_nephew)"}, + {"utterance": "who produces meet the press", "targetFormula": "(!fb:broadcast.content.producer fb:en.nbc_meet_the_press_video)"}, + {"utterance": "when did jack albertson die", "targetFormula": "(!fb:people.deceased_person.date_of_death fb:en.jack_albertson)"}, + {"utterance": "what tv programs has hugh laurie created", "targetFormula": "(fb:tv.tv_program.program_creator fb:en.hugh_laurie)"}, + {"utterance": "how many breeds are in the sporting group of the american kennel club", "targetFormula": "(count (!fb:biology.breed_registration.breed (and ((lambda x (fb:biology.breed_registration.breed_group (var x))) fb:en.sporting_group) ((lambda x (fb:biology.breed_registration.registry (var x))) fb:en.american_kennel_club))))"}, + {"utterance": "how many visitors does mammoth cave national park get in a year", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:protected_sites.protected_site.annual_visitors fb:en.mammoth_cave_national_park))"}, + {"utterance": "how many animals are there at the london zoo", "targetFormula": "(!fb:zoos.zoo.num_animals fb:en.london_zoo)"}, + {"utterance": "where is independence hall located", "targetFormula": "(!fb:location.location.containedby fb:en.independence_hall)"}, + {"utterance": "when was mark mckinney born", "targetFormula": "(!fb:people.person.date_of_birth fb:en.mark_mckinney)"}, + {"utterance": "when were ritz crackers introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.ritz_cracker))"}, + {"utterance": "when was tostitos introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.tostitos))"}, + {"utterance": "what are some periodicals on computer science", "targetFormula": "(!fb:book.periodical_subject.periodicals fb:en.computer_science)"}, + {"utterance": "in what year did motorola have the most revenue", "targetFormula": "(!fb:measurement_unit.dated_money_value.valid_date ((lambda x (!fb:business.business_operation.revenue (var x))) fb:en.motorola))"}, + {"utterance": "how many other names is prostate cancer known as", "targetFormula": "(count (!fb:common.topic.alias fb:en.prostate_cancer))"}, + {"utterance": "who are the creators of teenage mutant ninja turtles", "targetFormula": "(!fb:tv.tv_program.program_creator fb:m.053x52)"}, + {"utterance": "who proposed the fifth amendment", "targetFormula": "(!fb:law.constitutional_amendment.proposed_by fb:en.fifth_amendment_to_the_united_states_constitution)"}, + {"utterance": "what programming languages were used for aol instant messenger", "targetFormula": "(!fb:computer.software.languages_used fb:en.aol_instant_messenger)"}, + {"utterance": "how many awards did sesame street win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.sesame_street)))"}, + {"utterance": "in what division are the st. louis cardinals", "targetFormula": "(!fb:baseball.baseball_team.division fb:en.st_louis_cardinals)"}, + {"utterance": "what are the dna codons of glutamine", "targetFormula": "(!fb:biology.amino_acid.codons fb:en.glutamine)"}, + {"utterance": "how many us supreme court cases have there been", "targetFormula": "(count (!fb:law.court.legal_cases fb:en.supreme_court_of_the_united_states))"}, + {"utterance": "who is currently number 6 on the washington wizards", "targetFormula": "(!fb:basketball.basketball_roster_position.player (and ((lambda x (fb:basketball.basketball_roster_position.team (var x))) fb:en.washington_wizards) ((lambda x (fb:basketball.basketball_roster_position.number (var x))) (number 6.0 fb:en.unitless))))"}, + {"utterance": "what makes does ford have", "targetFormula": "(!fb:automotive.company.make_s fb:en.ford_motor_company)"}, + {"utterance": "what are some films on the apollo 11 mission", "targetFormula": "(fb:film.film.subjects fb:en.apollo_11)"}, + {"utterance": "what are some books about art deco", "targetFormula": "(!fb:book.book_subject.works fb:en.art_deco)"}, + {"utterance": "what is the official language of mauritania", "targetFormula": "(!fb:location.country.official_language fb:en.mauritania)"}, + {"utterance": "what cities does us interstate 5 pass", "targetFormula": "(!fb:transportation.road.major_cities fb:en.us_interstate_5)"}, + {"utterance": "how many people collect automobiles", "targetFormula": "(count (!fb:interests.collection.collector ((lambda x (fb:interests.collection.category (var x))) fb:en.automobile)))"}, + {"utterance": "how many parent types does igneous rock have", "targetFormula": "(count (!fb:geology.rock_type.parent_rock_type fb:en.igneous_rock))"}, + {"utterance": "what is the piston configuration of a ford model t engine", "targetFormula": "(!fb:engineering.piston_engine.piston_configuration fb:en.ford_model_t_engine)"}, + {"utterance": "what digital cameras take an sdhc card", "targetFormula": "(!fb:digicams.camera_storage_type.compatible_cameras fb:en.sdhc_card)"}, + {"utterance": "where is the uss alabama currently moored", "targetFormula": "(!fb:location.location.containedby fb:m.019zhn)"}, + {"utterance": "what are the divisions of the western conference of the nhl", "targetFormula": "(!fb:ice_hockey.hockey_conference.divisions fb:en.western_conference)"}, + {"utterance": "what were the highest wind speeds in hurricane katrina", "targetFormula": "(!fb:meteorology.tropical_cyclone.highest_winds fb:en.hurricane_katrina)"}, + {"utterance": "how long overall is the tahina", "targetFormula": "(!fb:boats.ship.length_overall fb:m.0b_h31m)"}, + {"utterance": "when was the mark vii monorail introduced", "targetFormula": "(!fb:rail.locomotive_class.introduced fb:en.mark_vii_monorail)"}, + {"utterance": "where is walden pond located", "targetFormula": "(!fb:location.location.containedby fb:en.walden_pond)"}, + {"utterance": "what diseases is fatigue a symptom of", "targetFormula": "(fb:medicine.disease.symptoms fb:en.fatigue)"}, + {"utterance": "what rock formations formed in the barremian period", "targetFormula": "(fb:geology.geological_formation.formed_during_period fb:en.barremian)"}, + {"utterance": "what is jcpenney 's operating income", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:business.business_operation.operating_income (var x))) fb:en.j_c_penney))"}, + {"utterance": "what types of judaism are there", "targetFormula": "(!fb:religion.religion.includes fb:en.judaism)"}, + {"utterance": "how many film performances did ron glass do", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.ron_glass)))"}, + {"utterance": "where was the uss constitution built", "targetFormula": "(!fb:boats.ship.place_built fb:en.uss_constitution)"}, + {"utterance": "what treatments are there for leukemia", "targetFormula": "(!fb:medicine.disease.treatments fb:en.leukemia)"}, + {"utterance": "what kind of website is bing", "targetFormula": "(!fb:internet.website.category fb:en.windows_live_search)"}, + {"utterance": "when did the standard school broadcast start production", "targetFormula": "(!fb:broadcast.content.production_start fb:en.the_standard_school_broadcast)"}, + {"utterance": "what are the rides at six flags america", "targetFormula": "(!fb:amusement_parks.park.rides fb:en.six_flags_america)"}, + {"utterance": "where did omarion 's musical career begin", "targetFormula": "(!fb:music.artist.origin fb:en.omarion_grandberry)"}, + {"utterance": "how many tracks has the band recorded", "targetFormula": "(count (!fb:music.artist.track fb:en.the_band))"}, + {"utterance": "what articles are in the july 1967 issue of frontier times", "targetFormula": "(!fb:book.contents.work ((lambda x (fb:book.contents.publication (var x))) fb:en.frontier_times_1967_july))"}, + {"utterance": "what species is spock", "targetFormula": "(!fb:fictional_universe.fictional_character.species fb:en.spock)"}, + {"utterance": "how many knights of the garter did the order of the garter have", "targetFormula": "(count (!fb:royalty.chivalric_order_membership.recipient (and ((lambda x (fb:royalty.chivalric_order_membership.order (var x))) fb:en.order_of_the_garter) ((lambda x (fb:royalty.chivalric_order_membership.title (var x))) fb:en.knight_of_the_garter))))"}, + {"utterance": "when did charlie_s angels come out", "targetFormula": "(!fb:film.film.initial_release_date fb:en.charlies_angels)"}, + {"utterance": "at what age to kids usually start second grade", "targetFormula": "(!fb:education.grade_level.typical_age_minimum fb:en.second_grade)"}, + {"utterance": "when was yosemite national park designated as a protected place", "targetFormula": "(!fb:protected_sites.protected_site.date_established fb:en.yosemite_national_park)"}, + {"utterance": "what substancs has robert downey jr abused", "targetFormula": "(!fb:celebrities.substance_abuse_problem.substance ((lambda x (fb:celebrities.substance_abuse_problem.celebrity (var x))) fb:en.robert_downey_jr))"}, + {"utterance": "what did elizabeth ii collect", "targetFormula": "(!fb:interests.collection.category ((lambda x (fb:interests.collection.collector (var x))) fb:en.elizabeth_ii_of_the_united_kingdom))"}, + {"utterance": "who owned doritos", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.doritos))"}, + {"utterance": "how many sub-types of metamorphic rock are there", "targetFormula": "(count (!fb:geology.rock_type.sub_types fb:en.metamorphic_rock))"}, + {"utterance": "what species is the doctor", "targetFormula": "(!fb:fictional_universe.fictional_character.species fb:en.the_doctor)"}, + {"utterance": "when did chips ahoy debut", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.chips_ahoy))"}, + {"utterance": "what is the price of a 2012 jeep wrangler sport", "targetFormula": "(!fb:measurement_unit.money_value.amount (!fb:automotive.trim_level.msrp fb:m.0hgwnr4))"}, + {"utterance": "how many nominations did nutty professor have", "targetFormula": "(count (!fb:award.award_nomination.award ((lambda x (fb:award.award_nomination.nominated_for (var x))) fb:en.the_nutty_professor_1996)))"}, + {"utterance": "what companies produce smartphones", "targetFormula": "(!fb:business.competitive_space_mediator.company ((lambda x (fb:business.competitive_space_mediator.space (var x))) fb:en.smartphone))"}, + {"utterance": "how many french operas are there", "targetFormula": "(count (fb:opera.opera.language fb:en.french))"}, + {"utterance": "what is the population of the people on the earth", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.earth))"}, + {"utterance": "what opera did jonathan miller direct", "targetFormula": "(!fb:opera.opera_director.operas_directed fb:en.jonathan_miller)"}, + {"utterance": "what are some types of hats", "targetFormula": "(!fb:fashion.garment.more_specialized_forms fb:en.hat)"}, + {"utterance": "when was scope introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:m.02r3cjp))"}, + {"utterance": "what award did jack albertson win in 1968", "targetFormula": "(!fb:award.award_honor.award (and ((lambda x (fb:award.award_honor.year (var x))) (date 1968 -1 -1)) ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.jack_albertson)))"}, + {"utterance": "who makes up tomkat", "targetFormula": "(!fb:celebrities.supercouple.partners fb:en.tomkat)"}, + {"utterance": "what area of disneyland is space mountain in", "targetFormula": "(!fb:amusement_parks.ride.area fb:en.space_mountain)"}, + {"utterance": "who was the music by in titanic", "targetFormula": "(!fb:film.film.music fb:en.titanic_special_edition_dvd)"}, + {"utterance": "what military branches use the f-5 freedom fighter", "targetFormula": "(!fb:aviation.aircraft_ownership_count.aircraft_owner ((lambda x (fb:aviation.aircraft_ownership_count.aircraft_model (var x))) fb:en.f-5_freedom_fighter))"}, + {"utterance": "what are the causes of syphilis", "targetFormula": "(!fb:medicine.disease.causes fb:en.syphillis)"}, + {"utterance": "what genre of music is b12", "targetFormula": "(!fb:music.artist.genre fb:en.b12)"}, + {"utterance": "who designs for bill blass limited", "targetFormula": "(!fb:fashion.designer_label_association.designer ((lambda x (fb:fashion.designer_label_association.label (var x))) fb:en.bill_blass_limited))"}, + {"utterance": "when did john f kennedy die", "targetFormula": "(!fb:people.deceased_person.date_of_death fb:en.john_f_kennedy)"}, + {"utterance": "who was vice president to woodrow wilson", "targetFormula": "(!fb:government.us_president.vice_president fb:en.woodrow_wilson)"}, + {"utterance": "when was borders founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.borders_group)"}, + {"utterance": "who fought in the the battle of the champions", "targetFormula": "(!fb:boxing.match_boxer_relationship.boxer ((lambda x (fb:boxing.match_boxer_relationship.match (var x))) fb:en.the_battle_of_the_champions))"}, + {"utterance": "who was the cohost in 1993 on cbs evening news", "targetFormula": "(!fb:tv.tv_regular_personal_appearance.person (and (and ((lambda x (fb:tv.tv_regular_personal_appearance.program (var x))) fb:en.cbs_evening_news) ((lambda x (fb:tv.tv_regular_personal_appearance.appearance_type (var x))) fb:en.co_host)) ((lambda x (fb:tv.tv_regular_personal_appearance.from (var x))) (date 1993 -1 -1))))"}, + {"utterance": "for what albums did barack obama win a grammy award for best spoken word album", "targetFormula": "(!fb:award.award_honor.honored_for (and ((lambda x (fb:award.award_honor.award (var x))) fb:en.grammy_award_for_best_spoken_word_album) ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.barack_obama)))"}, + {"utterance": "how many other names is omarion known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.omarion_grandberry))"}, + {"utterance": "what percent alcohol is a g.h. mumm cordon rouge brut", "targetFormula": "(!fb:wine.wine.percentage_alcohol fb:en.g_h_mumm_cordon_rouge_brut)"}, + {"utterance": "how many countries use the indian rupee", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.indian_rupee))"}, + {"utterance": "when was the uss croaker launched", "targetFormula": "(!fb:boats.ship.launched fb:en.uss_croaker)"}, + {"utterance": "who is the producer for 13 going on 30", "targetFormula": "(!fb:film.film.produced_by fb:en.13_going_on_30)"}, + {"utterance": "what is the australian dollar code", "targetFormula": "(!fb:finance.currency.currency_code fb:en.australian_dollar)"}, + {"utterance": "who composed the song ship of fools", "targetFormula": "(!fb:music.composition.composer fb:m.02knmd5)"}, + {"utterance": "what are some adaptations of the wonderful wizard of oz", "targetFormula": "(!fb:media_common.adapted_work.adaptations fb:en.the_wonderful_wizard_of_oz)"}, + {"utterance": "what generation of matter is a down quark", "targetFormula": "(!fb:physics.particle.generation fb:en.down_quark)"}, + {"utterance": "what is the max speed of kingda ka", "targetFormula": "(!fb:amusement_parks.ride.max_speed fb:en.kingda_ka)"}, + {"utterance": "what is ron glass 's place of birth", "targetFormula": "(!fb:people.person.place_of_birth fb:en.ron_glass)"}, + {"utterance": "who was rolling stones' greatest guitarist of all time in 2003", "targetFormula": "(!fb:award.ranking.item ( (lambda x (fb:award.ranking.list (var x))) fb:en.rolling_stones_100_greatest_guitarists_of_all_time))"}, + {"utterance": "what religion was pablo picasso", "targetFormula": "(!fb:people.person.religion fb:en.pablo_picasso)"}, + {"utterance": "who is the owner of easy cheese", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.easy_cheese))"}, + {"utterance": "which characters are playable in sonic rush", "targetFormula": "(!fb:cvg.game_performance.character ((lambda x (fb:cvg.game_performance.game (var x))) fb:en.sonic_rush))"}, + {"utterance": "who created the runaways comic book series", "targetFormula": "(!fb:comic_books.comic_book_series.created_by fb:en.runaways)"}, + {"utterance": "what textiles are made from cotton", "targetFormula": "(!fb:fashion.fiber.textiles_made_from_this_fiber fb:en.cotton)"}, + {"utterance": "what is the theme song of sesame street", "targetFormula": "(!fb:tv.tv_program.theme_song fb:en.sesame_street)"}, + {"utterance": "what movies won the golden globe award for best drama film", "targetFormula": "(!fb:award.award_honor.honored_for ( (lambda x (fb:award.award_honor.award (var x))) fb:en.golden_globe_award_for_best_motion_picture_-_drama))"}, + {"utterance": "what are the causes of a heart attack", "targetFormula": "(!fb:medicine.disease.causes fb:en.heart_failure)"}, + {"utterance": "how many main characters did charlie_s angels have", "targetFormula": "(count (!fb:film.performance.character ((lambda x (fb:film.performance.film (var x))) fb:en.charlies_angels)))"}, + {"utterance": "what ship class is the uss cobia", "targetFormula": "(!fb:boats.ship.ship_class fb:en.uss_cobia)"}, + {"utterance": "how many tv shows did jerry seinfeld produce", "targetFormula": "(count (!fb:tv.tv_producer_term.program ((lambda x (fb:tv.tv_producer_term.producer (var x))) fb:en.jerry_seinfeld)))"}, + {"utterance": "what rock formations are made of quartzite", "targetFormula": "(!fb:geology.rock_type.formations fb:en.quartzite)"}, + {"utterance": "what is rna binding", "targetFormula": "(!fb:biology.gene_ontology_group.description fb:en.rna_binding)"}, + {"utterance": "when did the band massive attack form", "targetFormula": "(!fb:music.artist.active_start fb:en.massive_attack)"}, + {"utterance": "when is the restoration of the rijksmuseum supposed to finish", "targetFormula": "(!fb:projects.project.planned_completion_date fb:m.0fq7hj3)"}, + {"utterance": "who founded gsusa", "targetFormula": "(!fb:organization.organization.founders fb:en.girl_scouts_of_the_usa)"}, + {"utterance": "how many products has mattel produced", "targetFormula": "(count (!fb:business.company_product_relationship.consumer_product (!fb:business.consumer_company.products fb:en.mattel)))"}, + {"utterance": "when was techvibes started", "targetFormula": "(!fb:internet.blog.started fb:en.techvibes)"}, + {"utterance": "who won the 1964 united states presidential election", "targetFormula": "(!fb:government.election.winner fb:en.united_states_presidential_election_1964)"}, + {"utterance": "how many countries are within north america", "targetFormula": "(count (fb:location.location.containedby fb:en.north_america))"}, + {"utterance": "how many films are there on antarctica", "targetFormula": "(count (fb:film.film.subjects fb:en.antarctica))"}, + {"utterance": "when was the green party founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.green_party)"}, + {"utterance": "when did the princetown railway close", "targetFormula": "(!fb:rail.railway.closed fb:en.princetown_railway)"}, + {"utterance": "who are the characters in peanuts", "targetFormula": "(!fb:comic_strips.comic_strip.characters fb:en.peanuts)"}, + {"utterance": "how much danish kroner is a us dollar worth", "targetFormula": "(!fb:finance.exchange_rate.target_of_exchange ((lambda x (fb:finance.exchange_rate.source_of_exchange (var x))) fb:en.danish_krone))"}, + {"utterance": "what series is a wrinkle in time a part of", "targetFormula": "(!fb:book.written_work.part_of_series fb:en.a_wrinkle_in_time)"}, + {"utterance": "how many soccer players are goalkeepers", "targetFormula": "(count (!fb:soccer.football_position.players fb:en.goalkeeper_football))"}, + {"utterance": "what software has oracle designed", "targetFormula": "(!fb:computer.software_developer.software fb:en.oracle_corporation)"}, + {"utterance": "how much time did alan shepard spend in space", "targetFormula": "(!fb:spaceflight.astronaut.space_time_minutes fb:en.alan_shepard)"}, + {"utterance": "how many hockey players play center", "targetFormula": "(count (!fb:ice_hockey.hockey_position.players fb:m.02qvdc))"}, + {"utterance": "who is wells fargo 's supplier", "targetFormula": "(!fb:business.customer.supplier fb:en.wells_fargo)"}, + {"utterance": "who were the members of the soul stirrers", "targetFormula": "(!fb:music.group_membership.member ((lambda x (fb:music.group_membership.group (var x))) fb:en.the_soul_stirrers))"}, + {"utterance": "where do the denver spurs play", "targetFormula": "(!fb:sports.sports_team.arena_stadium fb:en.denver_spurs)"}, + {"utterance": "who won best movie of 2010", "targetFormula": "(!fb:award.award_honor.award_winner (and ((lambda x (fb:award.award_honor.award (var x))) fb:en.academy_award_for_best_picture) ((lambda x (fb:award.award_honor.year (var x))) (date 2010 -1 -1))))"}, + {"utterance": "what hospitals specialize in cardiology", "targetFormula": "(!fb:medicine.medical_specialty.hospitals_with_this_specialty fb:en.cardiology)"}, + {"utterance": "when was hank aaron inducted into the hall of fame", "targetFormula": "(!fb:baseball.baseball_player.hall_of_fame_induction fb:en.henry_aaron)"}, + {"utterance": "what blog does robert scoble write", "targetFormula": "(!fb:internet.blogger.blog fb:en.robert_scoble)"}, + {"utterance": "when was girl scouts of the usa founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.girl_scouts_of_the_usa)"}, + {"utterance": "what is the atomic number of silicon", "targetFormula": "(!fb:chemistry.chemical_element.atomic_number fb:en.silicon)"}, + {"utterance": "when did the renaissance begin", "targetFormula": "(!fb:time.event.start_date fb:en.renaissance)"}, + {"utterance": "what year was danny devito born", "targetFormula": "(!fb:people.person.date_of_birth fb:en.danny_devito)"}, + {"utterance": "how much did transformers cost to produce", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:film.film.estimated_budget (var x))) fb:en.transformers))"}, + {"utterance": "where is the guardian circulated", "targetFormula": "(!fb:book.newspaper.circulation_areas fb:en.the_guardian)"}, + {"utterance": "what type of beer is miller lite", "targetFormula": "(!fb:food.beer.beer_style fb:en.miller_lite)"}, + {"utterance": "what is the temperature of polaris", "targetFormula": "(!fb:astronomy.star.temperature_k fb:m.0kjyrc7)"}, + {"utterance": "what are dave grohl 's musical genres", "targetFormula": "(!fb:music.artist.genre fb:en.dave_grohl)"}, + {"utterance": "what is currency code for the singapore dollar", "targetFormula": "(!fb:finance.currency.currency_code fb:en.singapore_dollar)"}, + {"utterance": "what movies won ascap film and television music awards for most performed songs from motion pictures", "targetFormula": "(!fb:award.award_honor.honored_for ((lambda x (fb:award.award_honor.award (var x))) fb:m.04d215m))"}, + {"utterance": "what is the scientific name for a horse", "targetFormula": "(!fb:biology.organism_classification.scientific_name fb:en.domesticated_horse)"}, + {"utterance": "what artists are influenced by lead belly", "targetFormula": "(!fb:influence.influence_node.influenced fb:en.leadbelly)"}, + {"utterance": "how many film productions did hayden panettiere appear in", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.hayden_panettiere)))"}, + {"utterance": "what is currency code for the swedish krona", "targetFormula": "(!fb:finance.currency.currency_code fb:en.swedish_krona)"}, + {"utterance": "who coaches the toronto maple leafs", "targetFormula": "(!fb:ice_hockey.hockey_team.coach fb:en.toronto_maple_leafs)"}, + {"utterance": "who was titanic produced by", "targetFormula": "(!fb:film.film.produced_by fb:en.titanic_special_edition_dvd)"}, + {"utterance": "how many topics are equivalent to city/town/village", "targetFormula": "(count (!fb:freebase.type_profile.equivalent_topic fb:location.citytown))"}, + {"utterance": "what are some cajun dishes", "targetFormula": "(!fb:dining.cuisine.dishes fb:m.01v7z)"}, + {"utterance": "who did the cover art for batman #477", "targetFormula": "(!fb:book.magazine_issue.cover_artist fb:en.batman_477)"}, + {"utterance": "how many countries use euros", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.euro))"}, + {"utterance": "how many storms were in the 2005 atlantic hurricane season", "targetFormula": "(!fb:meteorology.tropical_cyclone_season.total_storms fb:en.2005_atlantic_hurricane_season)"}, + {"utterance": "what was the strongest storm in the 1992 atlantic hurricane season", "targetFormula": "(!fb:meteorology.tropical_cyclone_season.strongest_storm fb:en.1992_atlantic_hurricane_season)"}, + {"utterance": "when did the apple i stop being sold", "targetFormula": "(!fb:computer.computer.discontinued fb:en.apple_i)"}, + {"utterance": "who has held the title of wba world champion", "targetFormula": "(!fb:boxing.boxing_title_tenure.champion ((lambda x (fb:boxing.boxing_title_tenure.title (var x))) fb:m.0chgh2j))"}, + {"utterance": "who studied martial arts under yip man", "targetFormula": "(!fb:martial_arts.martial_artist.martial_arts_students fb:en.yip_man)"}, + {"utterance": "how many losses has sean miller had in his career", "targetFormula": "(!fb:basketball.basketball_coach.season_losses fb:en.sean_miller_1968)"}, + {"utterance": "who wrote travels with my cello", "targetFormula": "(!fb:book.written_work.author fb:m.067y_k7)"}, + {"utterance": "who were the parents of ferdinand ii", "targetFormula": "(!fb:people.person.parents fb:en.ferdinand_ii_holy_roman_emperor)"}, + {"utterance": "who operates the silver star railway", "targetFormula": "(!fb:rail.railway_operator_relationship.operator ((lambda x (fb:rail.railway_operator_relationship.railway (var x))) fb:m.04rsv0))"}, + {"utterance": "what labels does sarah jessica parker wear", "targetFormula": "(!fb:base.popstra.fashion_choice.designer ((lambda x (fb:base.popstra.fashion_choice.fashion_wearer (var x))) fb:en.sarah_jessica_parker))"}, + {"utterance": "what awards has lost won", "targetFormula": "(!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.lost))"}, + {"utterance": "what ingredients are in italian cuisine", "targetFormula": "(!fb:dining.cuisine.ingredients fb:m.09y2k2)"}, + {"utterance": "what is the max speed of a collins class submarine", "targetFormula": "(!fb:boats.ship_class.max_speed_knots fb:en.collins_class_submarine)"}, + {"utterance": "what country does the royal navy belong to", "targetFormula": "(!fb:military.armed_force.military_combatant fb:en.navy_dept)"}, + {"utterance": "who played in the 1973 fa cup final", "targetFormula": "(!fb:soccer.football_match.teams fb:en.fa_cup_final_1973)"}, + {"utterance": "how many teams did jim thorpe play for", "targetFormula": "(count (!fb:american_football.football_historical_roster_position.team ( (lambda x (fb:american_football.football_historical_roster_position.player (var x))) fb:en.jim_thorpe)))"}, + {"utterance": "what are sacred sites in sunni islam", "targetFormula": "(!fb:religion.religion.sacred_sites fb:en.sunni_islam)"}, + {"utterance": "who produced country time", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.country_time))"}, + {"utterance": "what are some german beers", "targetFormula": "(!fb:food.beer_country_region.beers_from_here fb:en.germany)"}, + {"utterance": "what is the currency code for the indonesian rupiah", "targetFormula": "(!fb:finance.currency.currency_code fb:en.indonesian_rupiah)"}, + {"utterance": "what is sarah jessica parker's hobby", "targetFormula": "(!fb:interests.hobbyist.hobbies fb:en.sarah_jessica_parker)"}, + {"utterance": "what fictional characters have attended oxford", "targetFormula": "(!fb:fictional_universe.school_in_fiction.students_graduates fb:en.university_of_oxford)"}, + {"utterance": "how many people died in the boston molasses disaster", "targetFormula": "(!fb:event.disaster.fatalities fb:en.boston_molasses_disaster)"}, + {"utterance": "what position does keyon dooling play", "targetFormula": "(!fb:basketball.basketball_player.position_s fb:en.keyon_dooling)"}, + {"utterance": "how many oscar shows did frank sinatra host", "targetFormula": "(count (!fb:base.academyawards.host_of_oscar_show.oscar_show_hosted fb:en.frank_sinatra))"}, + {"utterance": "where was the cross of gold speech presented", "targetFormula": "(!fb:event.speech_or_presentation.event ((lambda x (fb:event.speech_or_presentation.presented_work (var x))) fb:en.cross_of_gold_speech))"}, + {"utterance": "when did hurricane andrew form", "targetFormula": "(!fb:meteorology.tropical_cyclone.formed fb:en.hurricane_andrew)"}, + {"utterance": "who designed the rolls-royce welland", "targetFormula": "(!fb:engineering.engine.designer fb:en.rolls-royce_welland)"}, + {"utterance": "how many cancer centers are comprehensive", "targetFormula": "(count (fb:medicine.cancer_center.comprehensive true))"}, + {"utterance": "how many albums has fleet foxes released", "targetFormula": "(count (!fb:music.artist.album fb:en.fleet_foxes))"}, + {"utterance": "what is the earth 's area", "targetFormula": "(!fb:location.location.area fb:en.earth)"}, + {"utterance": "how many other names are there for the usa_s currency", "targetFormula": "(count (!fb:common.topic.alias fb:en.us))"}, + {"utterance": "who has portrayed james bond in film", "targetFormula": "(!fb:film.performance.actor ((lambda x (fb:film.performance.character (var x))) fb:m.0clpml))"}, + {"utterance": "what is the slogan of bmw", "targetFormula": "(!fb:organization.organization.slogan fb:en.bmw)"}, + {"utterance": "when did robin williams stop abusing cocaine", "targetFormula": "(!fb:celebrities.substance_abuse_problem.end (and ((lambda x (fb:celebrities.substance_abuse_problem.celebrity (var x))) fb:en.robin_williams) ((lambda x (fb:celebrities.substance_abuse_problem.substance (var x))) fb:en.cocaine)))"}, + {"utterance": "how tall is the you only live once image", "targetFormula": "(!fb:common.image.size fb:m.0kk92b)"}, + {"utterance": "what is the voltage of a aa alkaline battery", "targetFormula": "(!fb:engineering.battery_size_cell_variation.voltage (and ((lambda x (fb:engineering.battery_size_cell_variation.cell_type (var x))) fb:en.alkaline_battery) ((lambda x (fb:engineering.battery_size_cell_variation.size (var x))) fb:en.aa_battery)))"}, + {"utterance": "how many stages of breast cancer are there", "targetFormula": "(count (!fb:medicine.disease.stages fb:en.breast_cancer))"}, + {"utterance": "what languages are spoken in firefly", "targetFormula": "(!fb:tv.tv_program.languages fb:m.014v3t)"}, + {"utterance": "when was the frida kahlo exhibit at the philadelphia art museum", "targetFormula": "(!fb:exhibitions.exhibition_run.opened_on (and ((lambda x (fb:exhibitions.exhibition_run.exhibition (var x))) fb:m.04d14h7) ((lambda x (fb:exhibitions.exhibition_run.venue (var x))) fb:en.philadelphia_museum_of_art)))"}, + {"utterance": "when was stove top stuffing introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.stove_top_stuffing))"}, + {"utterance": "how many units does the us army have", "targetFormula": "(count (!fb:military.armed_force.units fb:en.u_army))"}, + {"utterance": "who was the nutty professor produced by", "targetFormula": "(!fb:film.film.produced_by fb:en.the_nutty_professor_1996)"}, + {"utterance": "how many documentaries are there on netflix", "targetFormula": "(count (!fb:media_common.netflix_genre.titles fb:en.documentary))"}, + {"utterance": "who are the current senators of colorado", "targetFormula": "(!fb:government.government_position_held.office_holder (and ( (lambda x (fb:government.government_position_held.office_position_or_title (var x))) fb:en.united_states_senator) ((lambda x (fb:government.government_position_held.district_represented (var x))) fb:en.colorado)))"}, + {"utterance": "how many songs are there for guitar hero: aerosmith", "targetFormula": "(count (!fb:cvg.musical_game_song_relationship.song ((lambda x (fb:cvg.musical_game_song_relationship.game (var x))) fb:en.guitar_hero_aerosmith)))"}, + {"utterance": "what is lamarcus aldridge's number", "targetFormula": "(!fb:basketball.basketball_roster_position.number ((lambda x (fb:basketball.basketball_roster_position.player (var x))) fb:en.lamarcus_aldridge))"}, + {"utterance": "what are the sub-types of coal", "targetFormula": "(!fb:geology.rock_type.sub_types fb:en.coal)"}, + {"utterance": "how many books are there written on computer programming", "targetFormula": "(count (!fb:media_common.literary_genre.books_in_this_genre fb:en.computer_programming))"}, + {"utterance": "where did the rolling stones 2009 concert tour take place", "targetFormula": "(!fb:time.event.locations fb:en.the_rolling_stones_2009_concert_tour)"}, + {"utterance": "what is the electric charge of a proton", "targetFormula": "(!fb:physics.particle.electric_charge fb:en.proton)"}, + {"utterance": "when was macintosh introduced", "targetFormula": "(!fb:computer.computer.introduced fb:en.macintosh)"}, + {"utterance": "how many works did mozart dedicate to joseph haydn", "targetFormula": "(count (!fb:media_common.dedication.work_dedicated (and ((lambda x (fb:media_common.dedication.dedicated_to (var x))) fb:en.joseph_haydn) ((lambda x (fb:media_common.dedication.dedicated_by (var x))) fb:en.wolfgang_amadeus_mozart))))"}, + {"utterance": "what is the subject of 13 going on 30", "targetFormula": "(!fb:film.film.subjects fb:en.13_going_on_30)"}, + {"utterance": "who has influenced stephen fry", "targetFormula": "(!fb:influence.influence_node.influenced_by fb:en.stephen_fry)"}, + {"utterance": "who is the costume designer for bruce almighty", "targetFormula": "(!fb:film.film.costume_design_by fb:en.bruce_almighty)"}, + {"utterance": "who wrote the episode straight and true of the wire", "targetFormula": "(!fb:tv.tv_series_episode.writer fb:en.straight_and_true)"} +] diff --git a/freebase/data/tutorial-freebase.grammar b/freebase/data/tutorial-freebase.grammar new file mode 100644 index 0000000..8f23af8 --- /dev/null +++ b/freebase/data/tutorial-freebase.grammar @@ -0,0 +1,6 @@ +(rule $Unary ($PHRASE) (SimpleLexiconFn (type fb:type.any))) +(rule $Binary ($PHRASE) (SimpleLexiconFn (type (-> fb:type.any fb:type.any)))) +(rule $Set ($Unary) (IdentityFn)) +(rule $Set ($Unary $Set) (MergeFn and)) +(rule $Set ($Binary $Set) (JoinFn forward)) +(rule $ROOT ($Set) (IdentityFn)) diff --git a/data/tutorial.ttl b/freebase/data/tutorial.ttl similarity index 93% rename from data/tutorial.ttl rename to freebase/data/tutorial.ttl index fcc028c..9049edb 100644 --- a/data/tutorial.ttl +++ b/freebase/data/tutorial.ttl @@ -17,5 +17,5 @@ fb:en.mount_whitney fb:type.object.name "Mount Whitney"@en. fb:en.california fb:type.object.name "California"@en. fb:en.seattle fb:location.location.area "369.2"^^xsd:double. -fb:en.san_francisco fb:location.location.area "600.6"^^xsd:double. +fb:en.san_francisco fb:location.location.area "600.6"@en. fb:en.los_angeles fb:location.location.area "1301.97"^^xsd:double. diff --git a/data/unittest-learn-ccg.grammar b/freebase/data/unittest-learn-ccg.grammar similarity index 100% rename from data/unittest-learn-ccg.grammar rename to freebase/data/unittest-learn-ccg.grammar diff --git a/freebase/data/unittest-learn.examples b/freebase/data/unittest-learn.examples new file mode 100644 index 0000000..1ceaf13 --- /dev/null +++ b/freebase/data/unittest-learn.examples @@ -0,0 +1,9 @@ +(example + (utterance "American") + (targetFormula (fb:people.person.nationality fb:en.united_states)) +) + +(example + (utterance "American born in Paris") + (targetFormula (and (fb:people.person.nationality fb:en.united_states) (fb:people.person.place_of_birth fb:en.paris))) +) diff --git a/data/unittest-learn.grammar b/freebase/data/unittest-learn.grammar similarity index 100% rename from data/unittest-learn.grammar rename to freebase/data/unittest-learn.grammar diff --git a/scripts/extract-freebase-schema.rb b/freebase/scripts/extract-freebase-schema.rb similarity index 93% rename from scripts/extract-freebase-schema.rb rename to freebase/scripts/extract-freebase-schema.rb index 3283aad..9662308 100755 --- a/scripts/extract-freebase-schema.rb +++ b/freebase/scripts/extract-freebase-schema.rb @@ -3,7 +3,7 @@ require 'open-uri' if ARGV.size != 2 then - puts "Usage: " + puts "Usage: " exit 1 end $endpoint, $outPath = ARGV @@ -11,6 +11,7 @@ $endpoint, $outPath = ARGV # Return set of raw lines from the query (probably don't want to use this function directly) def makeQuery(query) queryUrl = URI::encode("PREFIX fb: #{query}") + #puts query `curl -s '#{$endpoint}?query=#{queryUrl}'`.split(/\n/) end @@ -90,6 +91,7 @@ end def putsTriple(out, x, p, y) out.puts [x, p, y].join("\t") + '.' + out.flush end def extractAll @@ -156,9 +158,16 @@ def extractAll 'fb:freebase.type_hints.mediator', 'fb:freebase.type_hints.included_types', nil].compact.each { |p| + unique = (p == 'fb:type.property.schema') || (p == 'fb:type.property.expected_type') + items = {} puts "Extracting #{p}" getEntries("select ?x ?y { ?x #{p} ?y }") { |x,y| next unless hasSupport[x] + if unique and items[x] + puts "ERROR: #{x} #{p} #{items[x]} already exists, trying to add #{y}, skipping..." + next + end + items[x] = y putsTriple(out, x, p, y) } } diff --git a/freebase/scripts/fbshell.rb b/freebase/scripts/fbshell.rb new file mode 100755 index 0000000..78609b2 --- /dev/null +++ b/freebase/scripts/fbshell.rb @@ -0,0 +1,210 @@ +#!/usr/bin/ruby + +require 'open-uri' +require 'json' +require 'nokogiri' +require 'socket' + +< query}] if query =~ /^fb:/ # Specify ID verbatim + raw = open('https://www.googleapis.com/freebase/v1/search?query=' + URI::encode(query)).read + json = JSON::parse(raw) + results = json['result'] + results.map { |r| + mid = r['mid'].gsub!(/^\//, 'fb:').sub!(/\//, '.') # /m/02mjmr => fb:m.02mjmr + id = r['id'] = call("get\t" + mid) # Canonicalize the ids to be consistent with our Freebase index + if not id + puts "Skipping entry with no id: #{r.inspect}" + nil + else + r + end + }.compact +end + +def sparql(query) + query = 'PREFIX fb: ' + query + ' LIMIT 1000' + encodedQuery = URI::encode(query).gsub(/\+/, '%2b') + queryUrl = "http://localhost:3093/sparql?query=#{encodedQuery}" + raw = open(queryUrl).read + raw.sub!(//, '') # Remove namespace + xml = Nokogiri::XML(raw) + def normalize(s) + s ? s.content.sub(/^http:\/\/rdf.freebase.com\/ns\//, 'fb:') : nil + end + xml.xpath('//result') +end + +def get_properties(entity) + query = "SELECT DISTINCT ?property ?property_name { #{entity} ?property ?value . ?property fb:type.object.name ?property_name }" + sparql(query).map { |r| + property = normalize(r.xpath('binding[@name="property"]/uri')[0]) + property_name = normalize(r.xpath('binding[@name="property_name"]/literal')[0]) + {'id' => entity, 'property' => property, 'property_name' => property_name} + } +end + +def get_values(entity, property) + query = "SELECT DISTINCT ?value ?value_name { #{entity} #{property} ?value . OPTIONAL { ?value fb:type.object.name ?value_name } }" + sparql(query).map { |r| + value = normalize(r.xpath('binding[@name="value"]/uri')[0]) + value = normalize(r.xpath('binding[@name="value"]/literal')[0]) if not value + value_name = normalize(r.xpath('binding[@name="value_name"]/literal')[0]) + {'id' => value, 'name' => value_name} + } +end + +def find_paths(item1, items2) + id1 = item1['id'] + id2 = items2[0]['id'] # TODO: expand + #p [id1, id2] + query = "SELECT DISTINCT ?property ?property_name { #{id1} ?property #{id2} . ?property fb:type.object.name ?property_name }" + one_hop = sparql(query).map { |r| + property = normalize(r.xpath('binding[@name="property"]/uri')[0]) + property_name = normalize(r.xpath('binding[@name="property_name"]/literal')[0]) + {'id' => id1, 'property' => property, 'property_name' => property_name} + } + query = "SELECT DISTINCT ?property1 ?property_name1 ?property2 ?property_name2 { #{id1} ?property1 ?x . ?x ?property2 #{id2} . ?property1 fb:type.object.name ?property_name1 . ?property2 fb:type.object.name ?property_name2 }" + two_hops = sparql(query).map { |r| + property1 = normalize(r.xpath('binding[@name="property1"]/uri')[0]) + property_name1 = normalize(r.xpath('binding[@name="property_name1"]/literal')[0]) + property2 = normalize(r.xpath('binding[@name="property2"]/uri')[0]) + property_name2 = normalize(r.xpath('binding[@name="property_name2"]/literal')[0]) + {'id' => id1, 'property' => property1 + '/' + property2, 'property_name' => property_name1 + ' / ' + property_name2} + } + one_hop + two_hops +end + +stack = [{'id' => 'fb:en.barack_obama'}] +#stack = [{'id' => 'fb:en.tom_cruise'}] +compute_list = lambda { |item| + if item['list'] + # Done + elsif item['target'] + item['list'] = find_paths(item, item['target']['list']) + elsif item['query'] + item['list'] = fbsearch(item['query']) + elsif item['property'] + item['list'] = get_values(item['id'], item['property']) + else + item['list'] = get_properties(item['id']) + end +} +push_item = lambda { |item| + compute_list.call(item) + stack << item +} +def render_values(values) + lim = 3 + s = values[0...lim].map{|x| [x['id'], x['name']].compact.join("_")}.join(' | ') + s += " (#{values.size})" if values.size > lim + s +end +print_list = lambda { |filter| + # Print list of items for the top + item = stack[-1] + compute_list.call(item) + item['list'].each_with_index { |r,i| + s = [i, r['property'] || r['id'], r['property_name'], r['name']].join("\t") + s += "\t" + render_values(r['list']) if r['list'] + puts s if (not filter) || s =~ /#{filter}/ + } +} +while true + item = stack[-1] + print [item['query'], item['id'], item['property'], item['target'] && item['target']['list'][0]['id']].compact.join(' ') + '> ' + line = gets + break if line == nil + line.chomp! + if line == '' + # List + print_list.call(nil) + elsif line =~ /^\/(.+)$/ + # List with filter + print_list.call($1) + elsif line == '..' + # Pop the stack + stack.pop if stack.size > 1 + print_list.call(nil) + elsif line == '!' + # call list on all children + compute_list.call(item) + item['list'].each { |r| + compute_list.call(r) + } + print_list.call(nil) + elsif line =~ /^(-)?(\d+)$/ + # Go to a particular selection of the list + stay = $1 + i = Integer($2) + compute_list.call(item) + if i < 0 || i >= item['list'].size + puts "Out of range (#{item['list'].size} items)" + else + push_item.call(item['list'][i]) + print_list.call(nil) + stack.pop if stay + end + elsif line =~ /^--(.+)$/ + # Find path to target + target = {'query' => $1} + compute_list.call(target) + if target['list'].size == 0 + puts 'No matches' + else + push_item.call({'id' => item['id'], 'target' => target}) + print_list.call(nil) + end + else + # Free form search form entities + push_item.call({'query' => line}) + print_list.call(nil) + end +end + +$mid2id.close diff --git a/scripts/virtuoso b/freebase/scripts/virtuoso similarity index 79% rename from scripts/virtuoso rename to freebase/scripts/virtuoso index 5811390..3121754 100755 --- a/scripts/virtuoso +++ b/freebase/scripts/virtuoso @@ -2,7 +2,7 @@ # This script provides a convenient wrapper for the Virtuoso SPARQL server. -$virtuosoPath = File.dirname($0)+"/../virtuoso-opensource" +$virtuosoPath = File.dirname($0)+"/../../virtuoso-opensource" if not File.exists?($virtuosoPath) puts "#{$virtuosoPath} does not exist" exit 1 @@ -27,8 +27,8 @@ def run(command) end def start - if ARGV.size != 2 - puts "Usage: " + if ARGV.size < 2 + puts "Usage: [memory (MB)]" exit 1 end dbPath = ARGV.shift @@ -37,7 +37,11 @@ def start Dir.mkdir(dbPath) if not File.exists?(dbPath) # Recommended: 70% of RAM, each buffer is 8K - memFree = parseInt(`cat /proc/meminfo | grep MemFree | awk '{print $2}'`) # KB + if ARGV[0] + memFree = Integer(ARGV[0]) * 1000 + else + memFree = parseInt(`cat /proc/meminfo | grep MemFree | awk '{print $2}'`) # KB + end numberOfBuffers = parseInt(memFree * 0.7 / 8) maxDirtyBuffers = numberOfBuffers / 2 puts "#{memFree} KB free, using #{numberOfBuffers} buffers, #{maxDirtyBuffers} dirty buffers" @@ -169,11 +173,22 @@ def add } end +def deleteAll + port = parseInt(ARGV.shift) + run "echo 'RDF_GLOBAL_RESET();' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}" +end + def stop port = parseInt(ARGV.shift) run "echo 'shutdown;' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}" end +def count + port = parseInt(ARGV.shift) + run "echo 'SPARQL SELECT COUNT(*) { ?s ?p ?o };' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}" + puts "If the database is empty, this number should be 2617." +end + def status port = parseInt(ARGV.shift) run "echo 'status();' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}" @@ -191,11 +206,13 @@ end if ARGV.size == 0 puts "Usage:" - puts " #{$0} start # Starts the server on the given port" - puts " #{$0} stop # Stop the server" - puts " #{$0} add [offset] # Add new triples to the server (from offset)" - puts " #{$0} status # Display server status" - puts " #{$0} shell # Open shell" + puts " #{$0} start [memory (MB)] # Starts the server on the given port" + puts " #{$0} stop # Stop the server" + puts " #{$0} add [offset] # Add new triples to the server (from offset)" + puts " #{$0} deleteAll # Delete the entire database (CAREFUL!)" + puts " #{$0} status # Display server status" + puts " #{$0} count # Display number of triples" + puts " #{$0} shell # Open shell" exit 1 end @@ -204,7 +221,9 @@ case command when 'start' then start when 'stop' then stop when 'add' then add + when 'deleteAll' then deleteAll when 'status' then status + when 'count' then count when 'shell' then shell else raise "Invalid command: #{command}" end diff --git a/module-classes.txt b/module-classes.txt new file mode 100644 index 0000000..428d580 --- /dev/null +++ b/module-classes.txt @@ -0,0 +1,361 @@ +cache edu.stanford.nlp.sempre.cache.FileStringCache +cache edu.stanford.nlp.sempre.cache.LruCallback +cache edu.stanford.nlp.sempre.cache.LruMap +cache edu.stanford.nlp.sempre.cache.RemoteStringCache +cache edu.stanford.nlp.sempre.cache.StringCache +cache edu.stanford.nlp.sempre.cache.StringCacheServer +cache edu.stanford.nlp.sempre.cache.StringCacheUtils +cache edu.stanford.nlp.sempre.cache.test.StringCacheTest +clojure edu.stanford.nlp.sempre.clojure.ClojureExecute +core edu.stanford.nlp.sempre.AbstractReinforcementParserState +core edu.stanford.nlp.sempre.AggregateFormula +core edu.stanford.nlp.sempre.ArithmeticFormula +core edu.stanford.nlp.sempre.AtomicSemType +core edu.stanford.nlp.sempre.BadFormulaException +core edu.stanford.nlp.sempre.BeamParser +core edu.stanford.nlp.sempre.BooleanValue +core edu.stanford.nlp.sempre.BoundedPriorityQueue +core edu.stanford.nlp.sempre.Builder +core edu.stanford.nlp.sempre.CallFormula +core edu.stanford.nlp.sempre.CallTypeInfo +core edu.stanford.nlp.sempre.CanonicalNames +core edu.stanford.nlp.sempre.ChartParserState +core edu.stanford.nlp.sempre.CoarseParser +core edu.stanford.nlp.sempre.Colorizer +core edu.stanford.nlp.sempre.ConcatFn +core edu.stanford.nlp.sempre.ConstantFn +core edu.stanford.nlp.sempre.ContextFn +core edu.stanford.nlp.sempre.ContextValue +core edu.stanford.nlp.sempre.Dataset +core edu.stanford.nlp.sempre.DateFn +core edu.stanford.nlp.sempre.DateValue +core edu.stanford.nlp.sempre.DerivOpCountFeatureComputer +core edu.stanford.nlp.sempre.Derivation +core edu.stanford.nlp.sempre.DerivationStream +core edu.stanford.nlp.sempre.DescriptionValue +core edu.stanford.nlp.sempre.ErrorValue +core edu.stanford.nlp.sempre.ExactValueEvaluator +core edu.stanford.nlp.sempre.Example +core edu.stanford.nlp.sempre.ExampleUtils +core edu.stanford.nlp.sempre.Executor +core edu.stanford.nlp.sempre.FeatureComputer +core edu.stanford.nlp.sempre.FeatureExtractor +core edu.stanford.nlp.sempre.FeatureMatcher +core edu.stanford.nlp.sempre.FeatureVector +core edu.stanford.nlp.sempre.FilterNerSpanFn +core edu.stanford.nlp.sempre.FilterPosTagFn +core edu.stanford.nlp.sempre.FilterSpanLengthFn +core edu.stanford.nlp.sempre.FloatingFeatureComputer +core edu.stanford.nlp.sempre.FloatingParser +core edu.stanford.nlp.sempre.FloatingRuleUtils +core edu.stanford.nlp.sempre.Formula +core edu.stanford.nlp.sempre.FormulaMatchExecutor +core edu.stanford.nlp.sempre.Formulas +core edu.stanford.nlp.sempre.FuncSemType +core edu.stanford.nlp.sempre.FuzzyMatchFn +core edu.stanford.nlp.sempre.Grammar +core edu.stanford.nlp.sempre.HasScore +core edu.stanford.nlp.sempre.IdentityFn +core edu.stanford.nlp.sempre.JavaExecutor +core edu.stanford.nlp.sempre.JoinFn +core edu.stanford.nlp.sempre.JoinFormula +core edu.stanford.nlp.sempre.Json +core edu.stanford.nlp.sempre.KnowledgeGraph +core edu.stanford.nlp.sempre.LambdaFormula +core edu.stanford.nlp.sempre.LanguageAnalyzer +core edu.stanford.nlp.sempre.LanguageInfo +core edu.stanford.nlp.sempre.Learner +core edu.stanford.nlp.sempre.ListValue +core edu.stanford.nlp.sempre.Main +core edu.stanford.nlp.sempre.MarkFormula +core edu.stanford.nlp.sempre.Master +core edu.stanford.nlp.sempre.MergeFn +core edu.stanford.nlp.sempre.MergeFormula +core edu.stanford.nlp.sempre.MultipleDerivationStream +core edu.stanford.nlp.sempre.NaiveKnowledgeGraph +core edu.stanford.nlp.sempre.NameValue +core edu.stanford.nlp.sempre.NotFormula +core edu.stanford.nlp.sempre.NullExecutor +core edu.stanford.nlp.sempre.NullTypeLookup +core edu.stanford.nlp.sempre.NumberFn +core edu.stanford.nlp.sempre.NumberValue +core edu.stanford.nlp.sempre.Params +core edu.stanford.nlp.sempre.ParaphraseModel +core edu.stanford.nlp.sempre.Parser +core edu.stanford.nlp.sempre.ParserAgenda +core edu.stanford.nlp.sempre.ParserState +core edu.stanford.nlp.sempre.PrimitiveFormula +core edu.stanford.nlp.sempre.ReinforcementParser +core edu.stanford.nlp.sempre.ReinforcementUtils +core edu.stanford.nlp.sempre.ReverseFormula +core edu.stanford.nlp.sempre.Rule +core edu.stanford.nlp.sempre.SelectFn +core edu.stanford.nlp.sempre.SemType +core edu.stanford.nlp.sempre.SemTypeHierarchy +core edu.stanford.nlp.sempre.SemanticFn +core edu.stanford.nlp.sempre.SempreUtils +core edu.stanford.nlp.sempre.Server +core edu.stanford.nlp.sempre.Session +core edu.stanford.nlp.sempre.SimpleAnalyzer +core edu.stanford.nlp.sempre.SimpleLexicon +core edu.stanford.nlp.sempre.SimpleLexiconFn +core edu.stanford.nlp.sempre.SingleDerivationStream +core edu.stanford.nlp.sempre.StringValue +core edu.stanford.nlp.sempre.SuperlativeFormula +core edu.stanford.nlp.sempre.TableValue +core edu.stanford.nlp.sempre.TopSemType +core edu.stanford.nlp.sempre.Trie +core edu.stanford.nlp.sempre.TypeInference +core edu.stanford.nlp.sempre.TypeLookup +core edu.stanford.nlp.sempre.UnionSemType +core edu.stanford.nlp.sempre.UriValue +core edu.stanford.nlp.sempre.Value +core edu.stanford.nlp.sempre.ValueEvaluator +core edu.stanford.nlp.sempre.ValueFormula +core edu.stanford.nlp.sempre.Values +core edu.stanford.nlp.sempre.VariableFormula +core edu.stanford.nlp.sempre.test.DerivationStreamTest +core edu.stanford.nlp.sempre.test.FormulaTest +core edu.stanford.nlp.sempre.test.GrammarTest +core edu.stanford.nlp.sempre.test.GrammarValidityTest +core edu.stanford.nlp.sempre.test.JavaExecutorTest +core edu.stanford.nlp.sempre.test.JsonTest +core edu.stanford.nlp.sempre.test.ParserTest +core edu.stanford.nlp.sempre.test.SemTypeTest +core edu.stanford.nlp.sempre.test.SemanticFnTest +core edu.stanford.nlp.sempre.test.SystemSanityTest +core edu.stanford.nlp.sempre.test.TestUtils +core edu.stanford.nlp.sempre.test.TypeInferenceTest +corenlp edu.stanford.nlp.sempre.corenlp.CoreNLPAnalyzer +corenlp edu.stanford.nlp.sempre.corenlp.test.CoreNLPSemanticFnTest +fbalignment edu.stanford.nlp.sempre.fbalignment.BinaryLexiconConstructionMain +fbalignment edu.stanford.nlp.sempre.fbalignment.FreebaseAlignmentDataManager +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.BipartiteBuilder +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.FromFbGraphBuilder +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.classify.ClassifierUtils +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.classify.DatumGenerator +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.BipartiteNodeFilter +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.FbRelationFilter +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.NodeFrequencyFilter +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.NodeFrequencyOrFbRelationFilter +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.AlignmentLearner +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.AlignmentLearnerFactory +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ChooseBestRelation +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ChooseBestTypeCompatibleRelation +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ExhaustiveNlTyping +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ExhaustiveNlTypingNoCommit +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.FbBasedNlTyping +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.NlIterLearner +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.SplitPredicateToDisjointTypes +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteEdge +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteEdgeFactory +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteGraph +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteNode +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteNodeFactory +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteNodeType +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.CompositeFbFact +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.CountEdge +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.FbBipartiteNode +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.FbTypedBipartiteNode +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.FourPartiteGraph +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.MatchSampleAndCountEdge +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.NlBipartiteNode +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.NlTypedBipartiteNode +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.ScoreEdge +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.UnlabeledEdge +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.CountScorer +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.MaxentScorer +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.NodePairScorer +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.ScorerFactory +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.SmoothedJaccardAndCountThresholdScorer +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.SmoothedJaccardScorer +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.AlignmentExample +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.AlignmentExperimentManager +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.AnnotatedAlignmentExample +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.HasLabel +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.PRCurve +fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.visualization.Visualizer +fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FbEntity +fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FbGraphBuilder +fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FbPropertiesExpectedTypes +fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FromPartiallyLinkedExtractionsGraphBuilder +fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FromPartiallyLinkedThroughCvtFbGraphBuilder +fbalignment edu.stanford.nlp.sempre.fbalignment.jung.DirectedSparseGraphVertexAccess +fbalignment edu.stanford.nlp.sempre.fbalignment.jung.UndirectedSparseGraphVertexAccess +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.ArgSubstringMatcher +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.ExactMatcher +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.IgnoreCaseAndTimeMatcher +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.IgnoreCaseMatcher +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.Matcher +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.MatcherFactory +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.SymmetricSubstringMatcher +fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.TimeMatcher +fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.BinaryExtractionsPreprocessor +fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.ExtractionsPreprocessor +fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.LinkedExtractionsPreprocessMain +fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.LinkedTimeExtractionCreator +fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.UnaryExtractionsPreprocessor +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.AddAlignmentScoresToPropertyInfoFile +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.AlignmentBinaryLexicon +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.EntityTypesAdder +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.ExtractResultsFromLogFile +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.FilterEntityFileByName +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateAtomicBinaryFormulaInfo +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateContainedByNameds +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateCvts +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateDataFromLog +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateEntityInfoFile +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateExpectedTypes +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GeneratePropertyFile +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateTypeToId +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateUnaryInfoFile +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.MidToIdFixer +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.MidToIdReplacer +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.SlashToDotConverter +fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.SplitExamples +fbalignment edu.stanford.nlp.sempre.fbalignment.testers.BipartiteGraphStatsTester +fbalignment edu.stanford.nlp.sempre.fbalignment.testers.FourPartiteLearnerTester +fbalignment edu.stanford.nlp.sempre.fbalignment.testers.LexiconTester +fbalignment edu.stanford.nlp.sempre.fbalignment.testers.LinearClassifierFromAnnotationFile +fbalignment edu.stanford.nlp.sempre.fbalignment.testers.UnaryInfoTester +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.FbUnaryAlignmentNode +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.FbUnaryNodeFilter +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.NlUnaryAlignmentNode +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryAlignmentManager +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryAlignmentNode +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryInfoData +fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryTupleGenerator +freebase edu.stanford.nlp.sempre.freebase.BinaryLexicon +freebase edu.stanford.nlp.sempre.freebase.BridgeFn +freebase edu.stanford.nlp.sempre.freebase.BuildCanonicalIdMap +freebase edu.stanford.nlp.sempre.freebase.BuildTypesMap +freebase edu.stanford.nlp.sempre.freebase.CanonicalizeExamples +freebase edu.stanford.nlp.sempre.freebase.CanonicalizeIds +freebase edu.stanford.nlp.sempre.freebase.EntityLexicon +freebase edu.stanford.nlp.sempre.freebase.ExecuteExamples +freebase edu.stanford.nlp.sempre.freebase.FbFormulasInfo +freebase edu.stanford.nlp.sempre.freebase.FilterFreebase +freebase edu.stanford.nlp.sempre.freebase.Free917Converter +freebase edu.stanford.nlp.sempre.freebase.FreebaseInfo +freebase edu.stanford.nlp.sempre.freebase.FreebaseSearch +freebase edu.stanford.nlp.sempre.freebase.FreebaseTypeLookup +freebase edu.stanford.nlp.sempre.freebase.FreebaseValueEvaluator +freebase edu.stanford.nlp.sempre.freebase.LambdaCalculusConverter +freebase edu.stanford.nlp.sempre.freebase.Lexicon +freebase edu.stanford.nlp.sempre.freebase.LexiconFn +freebase edu.stanford.nlp.sempre.freebase.SparqlExecutor +freebase edu.stanford.nlp.sempre.freebase.SparqlExpr +freebase edu.stanford.nlp.sempre.freebase.Stemmer +freebase edu.stanford.nlp.sempre.freebase.TextToTextMatcher +freebase edu.stanford.nlp.sempre.freebase.UnaryLexicon +freebase edu.stanford.nlp.sempre.freebase.Utils +freebase edu.stanford.nlp.sempre.freebase.index.FbEntityIndexer +freebase edu.stanford.nlp.sempre.freebase.index.FbEntitySearcher +freebase edu.stanford.nlp.sempre.freebase.index.FbIndexField +freebase edu.stanford.nlp.sempre.freebase.lexicons.EntrySource +freebase edu.stanford.nlp.sempre.freebase.lexicons.ExtremeValueWrapper +freebase edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry +freebase edu.stanford.nlp.sempre.freebase.lexicons.TokenLevelMatchFeatures +freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.BinaryNormalizer +freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.EntryNormalizer +freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.IdentityNormalizer +freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.PrepDropNormalizer +freebase edu.stanford.nlp.sempre.freebase.test.FbFormulasTest +freebase edu.stanford.nlp.sempre.freebase.test.FreebaseInfoTest +freebase edu.stanford.nlp.sempre.freebase.test.FreebaseSemTypeTest +freebase edu.stanford.nlp.sempre.freebase.test.FreebaseTypeInferenceTest +freebase edu.stanford.nlp.sempre.freebase.test.LambdaCalculusConverterTest +freebase edu.stanford.nlp.sempre.freebase.test.LexiconTest +freebase edu.stanford.nlp.sempre.freebase.test.PrepDropNormalizerTest +freebase edu.stanford.nlp.sempre.freebase.test.SparqlExecutorTest +freebase edu.stanford.nlp.sempre.freebase.test.StemmerTest +freebase edu.stanford.nlp.sempre.freebase.test.TokenMatchTest +freebase edu.stanford.nlp.sempre.freebase.utils.CollectionUtils +freebase edu.stanford.nlp.sempre.freebase.utils.DoubleContainer +freebase edu.stanford.nlp.sempre.freebase.utils.FileUtils +freebase edu.stanford.nlp.sempre.freebase.utils.FormatConverter +freebase edu.stanford.nlp.sempre.freebase.utils.FreebaseUtils +freebase edu.stanford.nlp.sempre.freebase.utils.LinkedExtractionFileUtils +freebase edu.stanford.nlp.sempre.freebase.utils.MathUtils +freebase edu.stanford.nlp.sempre.freebase.utils.SemparseLogTools +freebase edu.stanford.nlp.sempre.freebase.utils.ShortContainer +freebase edu.stanford.nlp.sempre.freebase.utils.WnExpander +freebase edu.stanford.nlp.sempre.freebase.utils.WordNet +jungle edu.stanford.nlp.sempre.jungle.WebformExecutor +paraphrase edu.stanford.nlp.sempre.paraphrase.Aligner +paraphrase edu.stanford.nlp.sempre.paraphrase.Context +paraphrase edu.stanford.nlp.sempre.paraphrase.DatasetUtils +paraphrase edu.stanford.nlp.sempre.paraphrase.FeatureSimilarity +paraphrase edu.stanford.nlp.sempre.paraphrase.FeatureSimilarityComputer +paraphrase edu.stanford.nlp.sempre.paraphrase.FormulaGenerationInfo +paraphrase edu.stanford.nlp.sempre.paraphrase.FormulaRetriever +paraphrase edu.stanford.nlp.sempre.paraphrase.Interval +paraphrase edu.stanford.nlp.sempre.paraphrase.NnBuilder +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseBuilder +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseDataset +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseDerivation +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseExample +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseFeatureExtractor +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseFeatureMatcher +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseLearner +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseMain +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseParser +paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseUtils +paraphrase edu.stanford.nlp.sempre.paraphrase.ParsingExample +paraphrase edu.stanford.nlp.sempre.paraphrase.ParsingExampleProcessor +paraphrase edu.stanford.nlp.sempre.paraphrase.Prediction +paraphrase edu.stanford.nlp.sempre.paraphrase.Proof +paraphrase edu.stanford.nlp.sempre.paraphrase.ProofScorer +paraphrase edu.stanford.nlp.sempre.paraphrase.ProofTransformationModel +paraphrase edu.stanford.nlp.sempre.paraphrase.QuestionGenerator +paraphrase edu.stanford.nlp.sempre.paraphrase.Transformer +paraphrase edu.stanford.nlp.sempre.paraphrase.VectorSpaceModel +paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParalexEditDistanceComputer +paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParalexQuestionReader +paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParalexRules +paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParaphraseTableExtractor +paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.PhraseTable +paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.PosNerTagger +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LangItem +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LanguageExp +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LanguageExpToken +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosRule +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosSequence +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.ParaphraseAlignment +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.RuleApplication +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.RuleApplier +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.Rulebase +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.SubstitutionRuleExtractor +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.SyntacticRuleSet +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.VerbSemClassExtractor +paraphrase edu.stanford.nlp.sempre.paraphrase.rules.VerbSemClassMatcher +paraphrase edu.stanford.nlp.sempre.paraphrase.scripts.CompareLogFiles +paraphrase edu.stanford.nlp.sempre.paraphrase.scripts.ExtractTrainTrueAndPredLines +paraphrase edu.stanford.nlp.sempre.paraphrase.scripts.GenerateOracleRelationsAndEntities +regex edu.stanford.nlp.sempre.regex.RegexValueEvaluator +tables edu.stanford.nlp.sempre.tables.ConvertCsvToTtl +tables edu.stanford.nlp.sempre.tables.DPParser +tables edu.stanford.nlp.sempre.tables.DPParserPruner +tables edu.stanford.nlp.sempre.tables.FuzzyMatcher +tables edu.stanford.nlp.sempre.tables.StringNormalizationUtils +tables edu.stanford.nlp.sempre.tables.TableKnowledgeGraph +tables edu.stanford.nlp.sempre.tables.TableTypeLookup +tables edu.stanford.nlp.sempre.tables.TableTypeSystem +tables edu.stanford.nlp.sempre.tables.TableValueEvaluator +tables edu.stanford.nlp.sempre.tables.lambdadcs.BinaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.BinaryTypeHint +tables edu.stanford.nlp.sempre.tables.lambdadcs.DenotationUtils +tables edu.stanford.nlp.sempre.tables.lambdadcs.ExplicitBinaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.ExplicitUnaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.InfiniteUnaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSException +tables edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSExecutor +tables edu.stanford.nlp.sempre.tables.lambdadcs.PredicateBinaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.SpecialBinaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.TypeHint +tables edu.stanford.nlp.sempre.tables.lambdadcs.UnaryDenotation +tables edu.stanford.nlp.sempre.tables.lambdadcs.UnaryTypeHint +tables edu.stanford.nlp.sempre.tables.test.AnnotatedFormulasChecker +tables edu.stanford.nlp.sempre.tables.test.CustomExample +tables edu.stanford.nlp.sempre.tables.test.DPParserChecker +tables edu.stanford.nlp.sempre.tables.test.LambdaDCSExecutorTest diff --git a/parasempre b/parasempre deleted file mode 100755 index c8e0627..0000000 --- a/parasempre +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/ruby - -$: << 'fig/lib' -require 'execrunner' - -# Note: run this on the NLP machines because SPARQL server is running on jack. -if ARGV.size == 0 - puts "Usage:" - puts " ./parasempre @mode=train @domain= @sparqlserver= @cacheserver=" - puts "Additional options can be any of the following:" - puts " - Additional program options (e.g., -BeamParser.beamSize 3)" - puts " - Execrunner options, which select the options to include (@data=0)" - exit 1 -end - -system "mkdir -p trans_state/execs" -system "touch trans_state/lastExec" - -def header - l( - letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')), - 'fig/bin/qcreate', - '-statePath', - 'trans_state', - (File.exists?('/u/nlp/bin/java7') ? '/u/nlp/bin/java7' : 'java'), - '-ea', - '-Xmx10g', - '-cp', 'classes:'+Dir['lib/*.jar'].join(':'), - letDefault(:prof, 0), sel(:prof, - l(), - '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt', - '-Xrunhprof:heap=sites,file=_OUTPATH_/java.hprof.txt'), - nil) -end - -def sparqlOpts - l( - required(:sparqlserver, 'host:port of the Sparql server'), - o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}), - nil) -end - -def defaultOpts - l( - o('execDir', '_OUTPATH_'), o('overwriteExecDir'), - o('addToView', 0), - nil) -end - -def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath) - l( - required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or : (hit the cacheserver)'), - lambda { |e| - cacheserver = e[:cacheserver] - case cacheserver - when 'none' then l() - when 'local' then l( # Use files directly - don't run more than one job that does this! - o('Lexicon.cachePath', 'LexiconFn.cache'), - o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'), - nil) - else l( - o('Lexicon.cachePath', cacheserver + ':' + lexiconFnCachePath), - o('SparqlExecutor.cachePath', cacheserver + ':' + sparqlExecutorCachePath), - nil) - end - }, - nil) -end - - -$defaultFeatureDomains = [ - 'Del', - 'Denotation', - 'Formula', - 'NamedEntity', - #'Pt', - 'Subst', - 'WhType', -nil].compact - -def unbalancedTrainDevSplit - l(o('ParaphraseDataset.trainFrac', 0.8), o('ParaphraseDataset.devFrac', 0.2)) -end - -def allTrainSplit - l(o('ParaphraseDataset.trainFrac', 1), o('ParaphraseDataset.devFrac', 0)) -end - -def train - l( - header, - 'edu.stanford.nlp.sempre.paraphrase.ParaphraseMain', - defaultOpts, sparqlOpts, - o('ParaphraseMain.mode','train'), - required(:domain, 'domain (webquestions or free917)'), - selectDomain, - o('Rulebase.ruleTypes','Pt','Syntax', 'Move', 'Subst'), - o('ParaphraseFeatureMatcher.featureDomains', *$defaultFeatureDomains), - o('Params.l1Reg','lazy'), - o('Aligner.useWordnet',true), - o('FeatureSimilarityComputer.mode','lexical_overlap'), - nil) -end - -def webquestions - l( - letDefault(:data, 0), - sel(:data, - l(o('ParaphraseDataset.parsingInPaths', - 'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'), - unbalancedTrainDevSplit, - nil), - l(o('ParaphraseDataset.parsingInPaths', - 'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json', - 'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'), - allTrainSplit, - nil), - nil), - o('ParaphraseLearner.maxTrainIters',2), - o('ParaphraseLearner.partialReward',true), - o('ParaphraseParser.alignment',true), - o('ParaphraseParser.vsm',true), - o('Lexicon.entitySearchStrategy','inexact'), - o('FormulaRetriever.supportCountUtterances',false), - o('FormulaRetriever.filterRelations',true), - o('FormulaRetriever.maxEntries',10), - o('FormulaRetriever.conservativeEntityExtraction',true), - o('Params.l1RegCoeff','0.00017782794'), - o('VectorSpaceModel.wordVectorFile','lib/wordreprs/cbow-lowercase-50.vectors'), - l( - o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/inexact/'), - cachePaths('LexiconFnWebQ.cache', 'SparqlExecutor.cache'), - nil), - nil) -end - -def free917 - l( - letDefault(:data, 0), - sel(:data, - l(o('ParaphraseDataset.parsingInPaths', 'train,data/free917.train.examples.canonicalized.json'), - unbalancedTrainDevSplit, - nil), - l(o('ParaphraseDataset.parsingInPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json'), - allTrainSplit, - nil), - nil), - o('ParaphraseLearner.maxTrainIters',10), - o('ParaphraseLearner.partialReward',false), - o('ParaphraseParser.alignment',true), - o('ParaphraseParser.vsm',true), - o('Lexicon.entitySearchStrategy','exact'), - o('FormulaRetriever.supportCountUtterances',true), - o('FormulaRetriever.filterRelations',false), - o('FormulaRetriever.maxEntries',1000), - o('FormulaRetriever.conservativeEntityExtraction',false), - o('Params.l1RegCoeff',0.00562341325), - o('VectorSpaceModel.wordVectorFile','lib/wordreprs/wordLexicon'), - l( - o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'), - cachePaths('LexiconFnFree917.cache', 'SparqlExecutor.cache'), - nil), - nil) -end - -def selectDomain - sel(:domain, { - 'webquestions' => webquestions, - 'free917' => free917, - }) -end - - -run!( - sel(:mode, { - 'train' => train, - }), -nil) diff --git a/pull-dependencies b/pull-dependencies new file mode 100755 index 0000000..ceb3a87 --- /dev/null +++ b/pull-dependencies @@ -0,0 +1,163 @@ +#!/usr/bin/env ruby + +# SEMPRE depends on several library/data files into *lib*. Run this script to +# copy those dependencies to your local directory. This allows you to run +# SEMPRE from anywhere. This file consists of a set of modules (which loosely +# correspond to the code modules). + +# The master copy of these dependencies are stored on the Stanford NLP +# machines. + +$version = '2.0' +$isPublic = true + + +def isZip(name) + # Directories are zipped + name.end_with?('.exec') or name !~ /\./ +end + +# - Download the dependencies from the Stanford SEMPRE servers. +def pull(sourcePath, dir=nil, opts={}) + puts sourcePath + destDir = 'lib' + (dir ? '/' + dir : '') + system "mkdir -p #{destDir}" + + name = File.basename(sourcePath) + ext = isZip(name) ? '.zip' : '' + + if $isPublic + # Download url => localPath + url = 'http://nlp.stanford.edu/software/sempre/dependencies-' + $version + sourcePath + ext + localPath = destDir + '/' + name + ext + system "mkdir -p #{File.dirname(localPath)}" or exit 1 + system "wget -c '#{url}' -O #{localPath}" or exit 1 + # Unzip localPath to destDir if it's a zip file + if isZip(name) + system "cd #{File.dirname(localPath)} && unzip #{File.basename(localPath)}" or exit 1 + system "rm #{localPath}" or exit 1 + end + else + end +end + +# source: path to master git repository +def updateGit(source) + dir = File.basename(source.sub(/\.git$/, '')) + if File.exists?(dir) + system 'cd '+dir+' && git pull' or exit 1 + else + system 'git clone ' + source or exit 1 + end +end + +def downloadExec(path) + ['options.map', 'params', 'grammar'].each { |file| + if File.exists?(path+'/'+file) + pull(path+'/'+file, 'models/'+File.basename(path)) + end + } +end + +$modules = [] +def addModule(name, description, func) + $modules << [name, description, func] +end + +############################################################ + +addModule('core', 'Core utilities (need to compile)', lambda { + # fig: options parsing, experiment management, utils + updateGit('https://github.com/percyliang/fig') + system 'make -C fig' or exit 1 + system 'mkdir -p lib && cd lib && ln -sf ../fig/fig.jar' or exit 1 + + # Google libraries + pull('/u/nlp/data/semparse/resources/guava-14.0.1.jar') + + # TestNG -- testing framework + pull('/u/nlp/data/semparse/resources/testng-6.8.5.jar') + pull('/u/nlp/data/semparse/resources/jcommander-1.30.jar') + + # Checkstyle: make sure code looks fine + pull('/u/nlp/data/semparse/resources/checkstyle') + + # JSON + pull('/u/nlp/data/semparse/resources/jackson-core-2.2.0.jar') + pull('/u/nlp/data/semparse/resources/jackson-annotations-2.2.0.jar') + pull('/u/nlp/data/semparse/resources/jackson-databind-2.2.0.jar') +}) + +addModule('corenlp', 'Stanford CoreNLP (code and modules)', lambda { + pull('/u/nlp/data/semparse/resources/stanford-corenlp-full-2013-06-20.zip', '', {:symlink => true}) + if not File.exists?('lib/stanford-corenlp-full-2013-06-20') + system "cd lib && unzip stanford-corenlp-full-2013-06-20.zip" or exit 1 + end + ['stanford-corenlp-3.2.0.jar', 'stanford-corenlp-3.2.0-models.jar', 'joda-time.jar', 'jollyday.jar'].each { |file| + system "ln -sfv stanford-corenlp-full-2013-06-20/#{file} lib" or exit 1 + } + pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2013-06-07-models.jar', '', {:symlink => true}) +}) + +addModule('freebase', 'Freebase: need to construct Freebase schemas', lambda { + # Freebase schema + pull('/u/nlp/data/semparse/scr/freebase/state/execs/93.exec/schema2.ttl', 'fb_data/93.exec') + + # Lucene libraries + pull('/u/nlp/data/semparse/resources/lucene-core-4.4.0.jar') + pull('/u/nlp/data/semparse/resources/lucene-analyzers-common-4.4.0.jar') + pull('/u/nlp/data/semparse/resources/lucene-queryparser-4.4.0.jar') + + # Freebase data (for lexicon) + pull('/u/nlp/data/semparse/scr/fb_data/7', 'fb_data') + + # WebQuestions dataset + pull('/u/nlp/data/semparse/webquestions/dataset_11/webquestions.examples.train.json', 'data/webquestions/dataset_11') + pull('/u/nlp/data/semparse/webquestions/dataset_11/webquestions.examples.test.json', 'data/webquestions/dataset_11') +}) + + + +addModule('virtuoso', 'Virtuoso: if want to run own SPARQL server locally', lambda { + updateGit('https://github.com/openlink/virtuoso-opensource') + # Run this command to compile: + #system "cd virtuoso-opensource && ./autogen.sh && ./configure --prefix=$PWD/install && make && make install" or exit 1 +}) + +addModule('fullfreebase-ttl', 'Freebase (ttl file)', lambda { + # This is just for your reference. It is not directly used by SEMPRE. + pull('/u/nlp/data/semparse/scr/freebase/state/execs/93.exec/0.ttl.bz2', 'fb_data/93.exec', {:symlink => true}) +}) + +addModule('fullfreebase-vdb', 'Freebase (Virtuoso database)', lambda { + # Virtuoso index of 0.ttl above. This is read (and written) by Virtuoso. + pull('/u/nlp/data/semparse/scr/freebase/state/execs/93.exec/vdb.tar.bz2', 'fb_data/93.exec', {:symlink => true}) + # You need to unzip this yourself and move these files to the right place. +}) + +addModule('fullfreebase-types', 'Freebase types', lambda { + # Map from mid (e.g., fb:m.02mjmr) to id (e.g., fb:en.barack_obama) (used to link external things like Freebase API search with our internal Freebase) + pull('/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map.gz', 'fb_data', {:symlink => true}) + # Map from id to types (used to do type inference on internal entities) + pull('/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized.en-types.gz', 'fb_data', {:symlink => true}) + # You need to unzip these yourself and move these files to the right place. +}) + + +############################################################ + +if ARGV.size == 0 + puts "#{$0} ... " + puts + puts "Modules:" + $modules.each { |name,description,func| + puts " #{name}: #{description}" + } +end + +$modules.each { |name,description,func| + if ARGV.index(name) + puts "===== Downloading #{name}: #{description}" + func.call + end +} diff --git a/release-acl2014.files b/release-acl2014.files deleted file mode 100644 index b7888fd..0000000 --- a/release-acl2014.files +++ /dev/null @@ -1,46 +0,0 @@ -# This file contains additional public datasets described in the EMNLP -# 2013 paper, including WebQuestions. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies emnlp2013 -# - -############################################################ -#wordnet -lib/wordnet-3.0-prolog -lib/derivation.txt - -#word vectors -lib/wordreprs/cbow-lowercase-50.vectors -lib/wordreprs/wordLexicon - -#paralex phrase table -lib/paralex/phrase-table.counts.txt - -# Lexicon alignment data -lib/fb_data/6/unaryInfoStringAndAlignment.txt -lib/fb_data/6/binaryInfoStringAndAlignment.txt - -############################################################ -# WebQuestions - -lib/data/webquestions/dataset_11/webquestions.examples.train.json -lib/data/webquestions/dataset_11/webquestions.examples.test.json - -############################################################ -# Free917 [Cai & Yates, 2013] - -data/free917.train.examples.canonicalized.json -data/free917.test.examples.canonicalized.json - -############################################################ -# Lucene -lib/lucene/4.4/free917 -lib/lucene/4.4/inexact - -############################################################ -# Model -lib/models/915.exec # test model for acl 2014 - diff --git a/release-code.files b/release-code.files deleted file mode 100644 index 6fe1576..0000000 --- a/release-code.files +++ /dev/null @@ -1,244 +0,0 @@ -# This file contains all the source files included in the public -# code release. -# -# These files go into the public github repo. - -############################################################ - -release-code.files -release-core.files -release-emnlp2013.files -release-emnlp2014.files -release-fullfreebase_ttl.files -release-fullfreebase_vdb.files -release-geofreebase_ttl.files -release-geofreebase_vdb.files - -sempre -parasempre -Makefile -README.md -LICENSE.txt -TUTORIAL.md -QUICKSTART.md -testng.xml -download-dependencies - -# Ruby utils: execrunner script etc. -fig/bin/qcreate -fig/bin/chunk -fig/lib/execrunner.rb -fig/lib/myutils.rb - -# Main -src/edu/stanford/nlp/sempre/Main.java -src/edu/stanford/nlp/sempre/Master.java -src/edu/stanford/nlp/sempre/Session.java -src/edu/stanford/nlp/sempre/Builder.java - -src/edu/stanford/nlp/sempre/Dataset.java -src/edu/stanford/nlp/sempre/Learner.java -src/edu/stanford/nlp/sempre/Params.java -src/edu/stanford/nlp/sempre/FeatureVector.java -src/edu/stanford/nlp/sempre/FeatureMatcher.java -src/edu/stanford/nlp/sempre/Evaluation.java - -# Parsing -src/edu/stanford/nlp/sempre/Parsers.java -src/edu/stanford/nlp/sempre/Parser.java -src/edu/stanford/nlp/sempre/ParserState.java -src/edu/stanford/nlp/sempre/BeamParser.java - -src/edu/stanford/nlp/sempre/Grammar.java -src/edu/stanford/nlp/sempre/Rule.java -src/edu/stanford/nlp/sempre/Derivation.java -src/edu/stanford/nlp/sempre/FeatureExtractor.java -src/edu/stanford/nlp/sempre/DerivationConstraint.java -src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java -src/edu/stanford/nlp/sempre/Example.java -src/edu/stanford/nlp/sempre/TextToTextMatcher.java -src/edu/stanford/nlp/sempre/FormulaRetriever.java -src/edu/stanford/nlp/sempre/FormulaGenerationInfo.java - -# Support -src/edu/stanford/nlp/sempre/LanguageInfo.java -src/edu/stanford/nlp/sempre/FbFormulasInfo.java -src/edu/stanford/nlp/sempre/FreebaseInfo.java -src/edu/stanford/nlp/sempre/StringCacheServer.java -src/edu/stanford/nlp/sempre/StringCacheUtils.java -src/edu/stanford/nlp/sempre/Json.java -src/edu/stanford/nlp/sempre/Stemmer.java -src/edu/stanford/nlp/sempre/StringCache.java -src/edu/stanford/nlp/sempre/Trie.java -src/edu/stanford/nlp/sempre/FreebaseSearch.java - -# Formula -src/edu/stanford/nlp/sempre/Formulas.java -src/edu/stanford/nlp/sempre/Formula.java -src/edu/stanford/nlp/sempre/PrimitiveFormula.java -src/edu/stanford/nlp/sempre/ValueFormula.java -src/edu/stanford/nlp/sempre/JoinFormula.java -src/edu/stanford/nlp/sempre/MergeFormula.java -src/edu/stanford/nlp/sempre/VariableFormula.java -src/edu/stanford/nlp/sempre/LambdaFormula.java -src/edu/stanford/nlp/sempre/MarkFormula.java -src/edu/stanford/nlp/sempre/ReverseFormula.java -src/edu/stanford/nlp/sempre/AggregateFormula.java -src/edu/stanford/nlp/sempre/SuperlativeFormula.java -src/edu/stanford/nlp/sempre/NotFormula.java -src/edu/stanford/nlp/sempre/CallFormula.java -src/edu/stanford/nlp/sempre/SemType.java - -# Value -src/edu/stanford/nlp/sempre/Values.java -src/edu/stanford/nlp/sempre/Value.java -src/edu/stanford/nlp/sempre/NumberValue.java -src/edu/stanford/nlp/sempre/StringValue.java -src/edu/stanford/nlp/sempre/DateValue.java -src/edu/stanford/nlp/sempre/UriValue.java -src/edu/stanford/nlp/sempre/ErrorValue.java -src/edu/stanford/nlp/sempre/ListValue.java -src/edu/stanford/nlp/sempre/NameValue.java -src/edu/stanford/nlp/sempre/BooleanValue.java -src/edu/stanford/nlp/sempre/DescriptionValue.java - -# SemanticFn -src/edu/stanford/nlp/sempre/SemanticFn.java -src/edu/stanford/nlp/sempre/IdentityFn.java -src/edu/stanford/nlp/sempre/ConstantFn.java -src/edu/stanford/nlp/sempre/NumberFn.java -src/edu/stanford/nlp/sempre/ConcatFn.java -src/edu/stanford/nlp/sempre/DateFn.java -src/edu/stanford/nlp/sempre/LexiconFn.java -src/edu/stanford/nlp/sempre/JoinFn.java -src/edu/stanford/nlp/sempre/MergeFn.java -src/edu/stanford/nlp/sempre/SelectFn.java -src/edu/stanford/nlp/sempre/FilterPosTagFn.java -src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java -src/edu/stanford/nlp/sempre/FilterNerSpanFn.java -src/edu/stanford/nlp/sempre/BridgeFn.java - -# Executor -src/edu/stanford/nlp/sempre/Executor.java -src/edu/stanford/nlp/sempre/NullExecutor.java -src/edu/stanford/nlp/sempre/JavaExecutor.java -src/edu/stanford/nlp/sempre/FormulaMatchExecutor.java -src/edu/stanford/nlp/sempre/SparqlExecutor.java -src/edu/stanford/nlp/sempre/SparqlExpr.java - -# Lexicon -src/edu/stanford/nlp/sempre/fbalignment/lexicons/Lexicon.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntityLexicon.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/BinaryLexicon.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/UnaryLexicon.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/LexicalEntry.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntrySource.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/ExtremeValueWrapper.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/TokenLevelMatchFeatures.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/WordDistance.java - -# Lexicon normalizers -src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/EntryNormalizer.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/IdentityNormalizer.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/PrepDropNormalizer.java -src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/BinaryNormalizer.java - -# Lucene indexing -src/edu/stanford/nlp/sempre/fbalignment/index/FbEntitySearcher.java -src/edu/stanford/nlp/sempre/fbalignment/index/FbEntityIndexer.java -src/edu/stanford/nlp/sempre/fbalignment/index/FbIndexField.java - -# FbAlignment -src/edu/stanford/nlp/sempre/fbalignment/utils/DoubleContainer.java -src/edu/stanford/nlp/sempre/fbalignment/utils/FileUtils.java -src/edu/stanford/nlp/sempre/fbalignment/utils/MathUtils.java -src/edu/stanford/nlp/sempre/fbalignment/utils/CollectionUtils.java -src/edu/stanford/nlp/sempre/fbalignment/utils/WnExpander.java -src/edu/stanford/nlp/sempre/fbalignment/utils/WordNet.java - -# Freebase -src/edu/stanford/nlp/sempre/freebase/Utils.java - -# Tests -src/edu/stanford/nlp/sempre/test/FbFormulasTest.java -src/edu/stanford/nlp/sempre/test/FormulaTest.java -src/edu/stanford/nlp/sempre/test/JavaExecutorTest.java -src/edu/stanford/nlp/sempre/test/JsonTest.java -src/edu/stanford/nlp/sempre/test/LexiconTest.java -src/edu/stanford/nlp/sempre/test/ParserTest.java -src/edu/stanford/nlp/sempre/test/PrepDropNormalizerTest.java -src/edu/stanford/nlp/sempre/test/SemTypeTest.java -src/edu/stanford/nlp/sempre/test/SemanticFnTest.java -src/edu/stanford/nlp/sempre/test/SparqlExecutorTest.java -src/edu/stanford/nlp/sempre/test/StemmerTest.java -src/edu/stanford/nlp/sempre/test/SystemSanityTest.java -src/edu/stanford/nlp/sempre/test/TestUtils.java -src/edu/stanford/nlp/sempre/test/TokenMatchTest.java - -# Vis -src/edu/stanford/nlp/sempre/Vis.java -src/edu/stanford/nlp/sempre/vis/BeamFigures.java -src/edu/stanford/nlp/sempre/vis/ConfusionMatrices.java -src/edu/stanford/nlp/sempre/vis/ExampleDerivations.java -src/edu/stanford/nlp/sempre/vis/Utils.java - -# Paraphrasing -src/edu/stanford/nlp/sempre/paraphrase/Aligner.java -src/edu/stanford/nlp/sempre/paraphrase/Context.java -src/edu/stanford/nlp/sempre/paraphrase/Context.java -src/edu/stanford/nlp/sempre/paraphrase/ContextModel.java -src/edu/stanford/nlp/sempre/paraphrase/ContextSimilarityModel.java -src/edu/stanford/nlp/sempre/paraphrase/DatasetUtils.java -src/edu/stanford/nlp/sempre/paraphrase/EntityInstance.java -src/edu/stanford/nlp/sempre/paraphrase/EntityModel.java -src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarity.java -src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarityComputer.java -src/edu/stanford/nlp/sempre/paraphrase/Interval.java -src/edu/stanford/nlp/sempre/paraphrase/NearestNeighborLearner.java -src/edu/stanford/nlp/sempre/paraphrase/NnBuilder.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseBuilder.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDataset.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDerivation.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseExample.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureExtractor.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureMatcher.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseLearner.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseMain.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseParser.java -src/edu/stanford/nlp/sempre/paraphrase/ParaphraseUtils.java -src/edu/stanford/nlp/sempre/paraphrase/ParsingExample.java -src/edu/stanford/nlp/sempre/paraphrase/ParsingExampleProcessor.java -src/edu/stanford/nlp/sempre/paraphrase/Prediction.java -src/edu/stanford/nlp/sempre/paraphrase/QuestionGenerator.java -src/edu/stanford/nlp/sempre/paraphrase/VectorSpaceModel.java -src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexQuestionReader.java -src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexRules.java -src/edu/stanford/nlp/sempre/paraphrase/paralex/PhraseTable.java -src/edu/stanford/nlp/sempre/paraphrase/rules/LangItem.java -src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExp.java -src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExpToken.java -src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosRule.java -src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosSequence.java -src/edu/stanford/nlp/sempre/paraphrase/rules/ParaphraseAlignment.java -src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplication.java -src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplier.java -src/edu/stanford/nlp/sempre/paraphrase/rules/Rulebase.java -src/edu/stanford/nlp/sempre/paraphrase/rules/SubstitutionRuleExtractor.java -src/edu/stanford/nlp/sempre/paraphrase/rules/SyntacticRuleSet.java -src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassExtractor.java -src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassMatcher.java - -# Create Freebase -scripts/extract-freebase-schema.rb -scripts/virtuoso - -############################################################ -# Really simple data - -data/unittest-learn.grammar -data/unittest-learn-ccg.grammar - -data/unittest-learn.examples.json - -data/tutorial.grammar -data/tutorial.examples.json diff --git a/release-core.files b/release-core.files deleted file mode 100644 index 13c18f4..0000000 --- a/release-core.files +++ /dev/null @@ -1,47 +0,0 @@ -# This file contains all the basic data and software dependencies on -# which the public code release relies to compile, build, and run in -# general. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies core -# - -############################################################ -# models / data - -lib/stanford-corenlp-3.2.0-models.jar -lib/joda-time.jar -lib/jollyday.jar -lib/stanford-corenlp-caseless-2013-06-07-models.jar - -# Freebase schema -lib/fb_data/93.exec/schema.ttl - -############################################################ -# 3rd-party libraries: needed to compile and run - -# Fig -lib/fig.jar - -# Google libraries -lib/guava-14.0.1.jar - -# Json -lib/jackson-core-2.2.0.jar -lib/jackson-annotations-2.2.0.jar -lib/jackson-databind-2.2.0.jar - -# Testing -lib/testng-6.8.5.jar -lib/jcommander-1.30.jar - -# Lucene -lib/lucene-core-4.4.0.jar -lib/lucene-analyzers-common-4.4.0.jar -lib/lucene-queryparser-4.4.0.jar - -# Stanford CoreNLP -lib/stanford-corenlp-3.2.0.jar - diff --git a/release-emnlp2013.files b/release-emnlp2013.files deleted file mode 100644 index f738c31..0000000 --- a/release-emnlp2013.files +++ /dev/null @@ -1,40 +0,0 @@ -# This file contains additional public datasets described in the EMNLP -# 2013 paper, including WebQuestions. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies emnlp2013 -# - -############################################################ -# Lexicon alignment data - -lib/fb_data/6/unaryInfoStringAndAlignment.txt -lib/fb_data/6/binaryInfoStringAndAlignment.txt - -############################################################ -# WebQuestions - -lib/data/webquestions/dataset_11/webquestions.examples.train.json -lib/data/webquestions/dataset_11/webquestions.examples.test.json - -############################################################ -# Free917 [Cai & Yates, 2013] - -data/free917.train.examples.canonicalized.json -data/free917.test.examples.canonicalized.json - -############################################################ -# Lucene -lib/lucene/4.4/free917 -lib/lucene/4.4/inexact - -############################################################ -# Model -lib/models/15.exec - -############################################################ -# Grammars -data/emnlp2013.exactent.grammar -data/emnlp2013.inexactent.grammar diff --git a/release-fullfreebase.files b/release-fullfreebase.files deleted file mode 100644 index eef0978..0000000 --- a/release-fullfreebase.files +++ /dev/null @@ -1,9 +0,0 @@ -# This file contains a raw ttl file and a Virtuoso index for all of Freebase. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies fullfreebase -# - -lib/freebase/93.exec diff --git a/release-fullfreebase_ttl.files b/release-fullfreebase_ttl.files deleted file mode 100644 index 91cd1a1..0000000 --- a/release-fullfreebase_ttl.files +++ /dev/null @@ -1,9 +0,0 @@ -# This file contains a raw ttl file for all of Freebase -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies fullfreebase_ttl -# - -lib/freebase/93.exec/0.ttl diff --git a/release-fullfreebase_vdb.files b/release-fullfreebase_vdb.files deleted file mode 100644 index 57081d5..0000000 --- a/release-fullfreebase_vdb.files +++ /dev/null @@ -1,9 +0,0 @@ -# This file contains Virtuoso index for all of Freebase. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies fullfreebase_vdb -# - -lib/freebase/93.exec/vdb diff --git a/release-geofreebase.files b/release-geofreebase.files deleted file mode 100644 index e06cb2c..0000000 --- a/release-geofreebase.files +++ /dev/null @@ -1,10 +0,0 @@ -# This file contains a raw ttl file and a Virtuoso index for a geographic -# subset of Freebase. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies geofreebase -# - -lib/freebase/98.exec diff --git a/release-geofreebase_ttl.files b/release-geofreebase_ttl.files deleted file mode 100644 index ea19d5d..0000000 --- a/release-geofreebase_ttl.files +++ /dev/null @@ -1,8 +0,0 @@ -# This file contains a raw ttl file for a geographic subset of Freebase. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies geofreebase_ttl -# -lib/freebase/98.exec/0.ttl diff --git a/release-geofreebase_vdb.files b/release-geofreebase_vdb.files deleted file mode 100644 index 5c6702e..0000000 --- a/release-geofreebase_vdb.files +++ /dev/null @@ -1,8 +0,0 @@ -# This file contains a Virtuoso index for a geographic subset of Freebase. -# -# These files are hosted on the Stanford NLP web endpoint and are -# intended to be obtained via the command -# -# ./download-dependencies geofreebase_vdb -# -lib/freebase/98.exec/vdb diff --git a/run b/run new file mode 100755 index 0000000..135f2f8 --- /dev/null +++ b/run @@ -0,0 +1,426 @@ +#!/usr/bin/ruby + +# This is the main entry point for running all SEMPRE programs. See +# fig/lib/execrunner.rb for more documentation for how commands are generated. +# There are a bunch of modes that this script can be invoked with, which +# loosely correspond to the modules. + +$: << 'fig/lib' +require 'execrunner' + +$modes = [] +def addMode(name, description, func) + $modes << [name, description, func] +end + +def header(modules='core') + l( + # Queuing system + letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')), + # Create execution directory + 'fig/bin/qcreate', + # Run the Java command... + 'java', + '-ea', + '-Dmodules='+modules, + '-Xmx10g', + '-cp', 'libsempre/*:lib/*', + # Profiling + letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'), + nil) +end + +def rlwrap; system('which rlwrap') ? 'rlwrap' : nil end + +def unbalancedTrainDevSplit + l(o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2)) +end +def balancedTrainDevSplit + l(o('Dataset.trainFrac', 0.5), o('Dataset.devFrac', 0.5)) +end + +def figOpts; l(o('execDir', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end + +############################################################ +# Unit tests + +addMode('test', 'Run unit tests', lambda { |e| + l( + 'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*', 'org.testng.TestNG', + lambda { |e| + if e[:class] + l('-testclass', 'edu.stanford.nlp.sempre.' + e[:class]) + else + 'testng.xml' + end + }, + lambda { |e| + if e[:fast] + o('excludegroups', 'sparql,corenlp') + else + nil + end + }, + nil) +}) + +############################################################ +# Freebase + +def freebaseHeader; header('core,freebase') end + +def freebaseFeatureDomains + [ + 'basicStats', + 'alignmentScores', + 'entityFeatures', + 'context', + 'skipPos', + 'joinPos', + 'wordSim', + 'lexAlign', + 'tokenMatch', + 'rule', + 'opCount', + 'constant', + 'denotation', + 'whType', + 'span', + 'derivRank', + 'lemmaAndBinaries', + nil].compact +end + +def sparqlOpts + l( + required(:sparqlserver, 'host:port of the Sparql server'), # Example: jonsson:3093, etc. + o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}), + nil) +end + +def freebaseOpts + l( + figOpts, + sparqlOpts, + + # Features + o('FeatureExtractor.featureDomains', *freebaseFeatureDomains), + o('Builder.executor', 'freebase.SparqlExecutor'), + o('Builder.valueEvaluator', 'freebase.FreebaseValueEvaluator'), + o('LanguageAnalyzer.languageAnalyzer', 'corenlp.CoreNLPAnalyzer'), + + # Lexicon + o('LexiconFn.lexiconClassName', 'edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon'), + l( # binary + o('BinaryLexicon.binaryLexiconFilesPath', 'lib/fb_data/7/binaryInfoStringAndAlignment.txt'), + o('BinaryLexicon.keyToSortBy', 'Intersection_size_typed'), + nil), + o('UnaryLexicon.unaryLexiconFilePath', 'lib/fb_data/7/unaryInfoStringAndAlignment.txt'), # unary + o('EntityLexicon.entityPopularityPath', 'lib/fb_data/7/entityPopularity.txt'), # entity + nil) +end + +def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath) + l( + required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or : (hit the cacheserver)'), + lambda { |e| + cacheserver = e[:cacheserver] + cacheserver = 'jonsson:4000' if cacheserver == 'remote' # Default + case cacheserver + when 'none' then l() + when 'local' then l( # Use files directly - don't run more than one job that does this! + o('Lexicon.cachePath', 'LexiconFn.cache'), + o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'), + nil) + else l( + o('Lexicon.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+lexiconFnCachePath), + o('SparqlExecutor.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+sparqlExecutorCachePath), + nil) + end + }, + nil) +end + +# tag is either "free917" or "webquestions" +def emnlp2013AblationExperiments(tag) + l( + letDefault(:ablation, 0), + # Ablation experiments (EMNLP) + sel(:ablation, + l(), # (0) Just run things normally + selo(nil, 'Parser.beamSize', 200, 50, 10), # (1) Vary beam size + selo(nil, 'Dataset.trainFrac', 0.1, 0.2, 0.4, 0.6), # (2) Vary training set size + sel(nil, # (3) Structural: only do join or only do bridge + o('Grammar.tags', l(tag, 'join')), + o('Grammar.tags', l(tag, 'bridge')), + o('Grammar.tags', l(tag, 'inject')), + nil), + sel(nil, # (4) Features + o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains+['lexAlign'])), # +lexAlign + o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains+['lexAlign']-['alignmentScores'])), # +lexAlign -alignmentScores + o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains-['denotation'])), # -denotation + o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains-['skipPos', 'joinPos'])), # -syntax features (skipPos, joinPos) + nil), + #o('Builder.executor', 'FormulaMatchExecutor'), # (6) train on logical forms (doesn't really work well) + nil), + + letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1, 2, 3), + nil) +end + +def free917 + l( # Data + letDefault(:data, 0), + sel(:data, + l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2 + l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json')), # (1) Don't run on test yet! + nil), + + # Grammar + o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'), + o('Parser.beamSize', 500), + + emnlp2013AblationExperiments('free917'), + + # lexicon index + letDefault(:lucene, 0), + sel(:lucene, + l( + o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'), + cachePaths('10/LexiconFn.cache', '10/SparqlExecutor.cache'), + o('Grammar.tags', 'free917', 'bridge', 'join', 'inject', 'exact'), + nil), + l( # With entity disambiguation - currently too crappy + o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'), + cachePaths('4/LexiconFn.cache', '4/SparqlExecutor.cache'), + o('Grammar.tags', 'free917', 'bridge', 'join', 'inject', 'inexact'), + nil), + nil), + # Use binary predicate features (overfits on free917) + o('BridgeFn.filterBadDomain',false), + # Learning + o('Learner.maxTrainIters', 6), + nil) +end + +def webquestions + l( + # Data + letDefault(:data, 0), + sel(:data, + l( # Webquestions (dev) [EMNLP final JSON] + o('Dataset.inPaths', + 'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'), + unbalancedTrainDevSplit, + nil), + l( # Webquestions (test) [EMNLP final JSON] + o('Dataset.inPaths', + 'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json', + 'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'), + nil), + nil), + + # Grammar + o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'), + + o('Parser.beamSize', 200), # {07/03/13}: WebQuestions is too slow to run with default 500, so set to 200 for now... + + # Caching + letDefault(:entitysearch, 1), + sel(:entitysearch, # Used for EMNLP 2013 + l( + cachePaths('lucene/0.cache', 'sparql/1.cache'), + o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'), + o('LexiconFn.maxEntityEntries',10), + o('Grammar.tags', 'webquestions', 'bridge', 'join', 'inject','inexact'), # specify also strategy + nil), + nil), + + # Learning + o('Learner.maxTrainIters', 3), + + # Use binary predicate features (overfits on free917) + o('BridgeFn.useBinaryPredicateFeatures', true), + o('BridgeFn.filterBadDomain',true), + letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1,2,3), + nil) +end + + +addMode('freebase', 'Freebase (for EMNLP 2013, ACL 2014, TACL 2014)', lambda { |e| l( + letDefault(:train, 0), + letDefault(:interact, 0), + + sel(:interact, l(), rlwrap), + freebaseHeader, + 'edu.stanford.nlp.sempre.Main', + freebaseOpts, + + # Dataset + sel(:domain, { + 'webquestions' => webquestions, + 'free917' => free917, + }), + + # Training + sel(:train, l(), l( + letDefault(:agenda, 0), + sel(:agenda, l(), agendaExperiments, agendaFree917Experiments), + nil)), + + sel(:interact, l(), l( + # After training, run interact, which loads up a set of parameters and + # puts you in a prompt. + o('Dataset.inPaths'), + o('Learner.maxTrainIters', 0), + required(:load, 'none or exec number (e.g., 15) to load'), + lambda { |e| + if e[:load] == 'none' then + l() + else + execPath = "lib/models/#{e[:load]}.exec" + l( + o('Builder.inParamsPath', execPath+'/params'), + o('Grammar.inPaths', execPath+'/grammar'), + o('Master.logPath', lambda{|e| 'state/' + e[:domain] + '.log'}), + o('Master.newExamplesPath', lambda{|e| 'state/' + e[:domain] + '.examples'}), + o('Master.onlineLearnExamples', true), + # Make sure features are set properly! + nil) + end + }, + o('Main.interactive'), + nil)) +) }) + +addMode('cacheserver', 'Start the general-purpose cache server that serves files with key-value maps', lambda { |e| + l( + 'java', '-Xmx36g', '-ea', '-cp', 'libsempre/*:lib/fig.jar', + 'edu.stanford.nlp.sempre.cache.StringCacheServer', + letDefault(:port, 4000), + lambda { |e| o('port', e[:port]) }, + + letDefault(:cachetype, 1), + sel(:cachetype, + l( + o('FileStringCache.appendMode'), + o('FileStringCache.capacity', 35 * 1024), + o('FileStringCache.flushFrequency', 2147483647), + nil), + l( + o('FileStringCache.appendMode',false), + o('FileStringCache.capacity', 1 * 1024), + o('FileStringCache.flushFrequency', 100000), + nil), + nil), + nil) +}) + +############################################################ +# Freebase RDF database (for building SPARQL database) + +# Scratch directory +def scrOptions + letDefault(:scr, '/u/nlp/data/semparse/rdf/scr/' + `hostname | cut -f 1 -d .`.chomp) +end + +addMode('filterfreebase', '(1) Filter RDF Freebase dump (do this once) [takes about 1 hour]', lambda { |e| l( + scrOptions, + l( + 'fig/bin/qcreate', o('statePath', lambda{|e| e[:scr] + '/state'}), + 'java', '-ea', '-Xmx20g', '-cp', 'libsempre/*:lib/*', + 'edu.stanford.nlp.sempre.freebase.FilterFreebase', + o('inPath', '/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized'), + sel(:keep, { + 'all' => o('keepAllProperties'), + 'geo' => l( + o('keepTypesPaths', 'data/geo.types'), + o('keepPropertiesPath', 'data/geo.properties'), + o('keepGeneralPropertiesOnlyForSeenEntities', true), + nil), + }), + o('execDir', '_OUTPATH_'), o('overwriteExecDir'), + nil), +nil) }) + +addMode('sparqlserver', '(2) Start the SPARQL server [do this every time]', lambda { |e| l( + scrOptions, + required(:exec), + 'freebase/scripts/virtuoso', 'start', + lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory + lambda{|e| 3000+e[:exec]}, # port +nil) }) + +# (3) Index the filtered RDF dump [takes 48 hours] +addMode('indexfreebase', '(3) Index the filtered RDF dump [takes 48 hours for Freebase]', lambda { |e| l( + letDefault(:stage, nil), + scrOptions, + required(:exec), + sel(:stage, + l( + 'scripts/virtuoso', 'add', + lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/0.ttl'}, # ttl file + lambda{|e| 3000+e[:exec]}, # port + lambda{|e| e[:offset] || 0}, # offset + nil), + l( + 'scripts/extract-freebase-schema.rb', + lambda{|e| 'http://localhost:'+(3000+e[:exec]).to_s+'/sparql'}, # port + lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/schema.ttl'}, + nil), + nil), +nil) }) + +addMode('convertfree917', 'Convert the Free917 dataset', lambda { |e| l( + 'java', '-ea', '-Xmx15g', + '-cp', 'libsempre/*:lib/*', + 'edu.stanford.nlp.sempre.freebase.Free917Converter', + o('inDir','/u/nlp/data/semparse/yates/final-dataset-acl-2013-all/'), + o('outDir','data/free917_convert/'), + o('entityInfoFile','/user/joberant/scr/fb_data/3/entityInfo.txt'), + o('cvtFile','lib/fb_data/2/Cvts.txt'), + o('midToIdFile','/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map'), +nil) }) + +addMode('query', 'Query a single logical form or SPARQL', lambda { |e| l( + 'java', '-ea', + '-cp', 'libsempre/*:lib/*', + 'edu.stanford.nlp.sempre.freebase.SparqlExecutor', + sparqlOpts, +nil) }) + +############################################################ + +# Just start a simple interactive shell to try out SEMPRE commands +addMode('simple', 'Simple shell', lambda { |e| l( + rlwrap, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main', + o('interactive'), +nil) }) + +addMode('simple-sparql', 'Simple shell for querying SPARQL', lambda { |e| l( + rlwrap, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main', + o('executor', 'freebase.SparqlExecutor'), + sparqlOpts, + o('interactive'), +nil) }) + + + + +############################################################ + +if ARGV.size == 0 + puts "#{$0} @mode= [options]" + puts + puts 'This is the main entry point for all SEMPRE-related runs.' + puts "Modes:" + $modes.each { |name,description,func| + puts " #{name}: #{description}" + } +end + +modesMap = {} +$modes.each { |name,description,func| + modesMap[name] = func +} +run!(sel(:mode, modesMap)) diff --git a/scripts/agenda-stats b/scripts/agenda-stats new file mode 100755 index 0000000..34a529c --- /dev/null +++ b/scripts/agenda-stats @@ -0,0 +1,7 @@ +#!/usr/bin/ruby + +ARGV.each { |e| + e = e.sub(/\.exec$/, '') + puts "===== #{e}" + system "./fig/bin/tab -s -H -i e/#{e}.exec/learner.events -a iter group .sort utterance / ^parseTime$ ^numOfFeaturizedDerivs$ ^firstCorrectItem$ ^totalDerivs$ ^partCorrect$ ^partOracle$" +} diff --git a/scripts/checkstyle.sh b/scripts/checkstyle.sh new file mode 100755 index 0000000..d605fd6 --- /dev/null +++ b/scripts/checkstyle.sh @@ -0,0 +1,10 @@ +# Check style of the code + +if [ -z "$1" ]; then + files=`find src -name "*.java"` +else + files="$@" +fi + +d=`dirname $0` +java -cp $d/../lib/checkstyle/checkstyle-6.1.1-all.jar com.puppycrawl.tools.checkstyle.Main -c `dirname $0`/checkstyle.xml $files diff --git a/scripts/checkstyle.xml b/scripts/checkstyle.xml new file mode 100644 index 0000000..446e998 --- /dev/null +++ b/scripts/checkstyle.xml @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/evaluation.py b/scripts/evaluation.py new file mode 100755 index 0000000..b7f298a --- /dev/null +++ b/scripts/evaluation.py @@ -0,0 +1,67 @@ +#!/usr/bin/python + +import sys +import json + +# Official evaluation script used to evaluate Freebase question answering +# systems. Used for EMNLP 2013, ACL 2014 papers, etc. + +if len(sys.argv) != 2: + sys.exit("Usage: %s " % sys.argv[0]) + +"""return a tuple with recall, precision, and f1 for one example""" +def computeF1(goldList,predictedList): + + """Assume all questions have at least one answer""" + if len(goldList)==0: + raise Exception("gold list may not be empty") + """If we return an empty list recall is zero and precision is one""" + if len(predictedList)==0: + return (0,1,0) + """It is guaranteed now that both lists are not empty""" + + precision = 0 + for entity in predictedList: + if entity in goldList: + precision+=1 + precision = float(precision) / len(predictedList) + + recall=0 + for entity in goldList: + if entity in predictedList: + recall+=1 + recall = float(recall) / len(goldList) + + f1 = 0 + if precision+recall>0: + f1 = 2*recall*precision / (precision + recall) + return (recall,precision,f1) + +averageRecall=0 +averagePrecision=0 +averageF1=0 +count=0 + +"""Go over all lines and compute recall, precision and F1""" +with open(sys.argv[1]) as f: + for line in f: + tokens = line.split("\t") + gold = json.loads(tokens[1]) + predicted = json.loads(tokens[2]) + recall, precision, f1 = computeF1(gold,predicted) + averageRecall += recall + averagePrecision += precision + averageF1 += f1 + count+=1 + +"""Print final results""" +averageRecall = float(averageRecall) / count +averagePrecision = float(averagePrecision) / count +averageF1 = float(averageF1) / count +print "Number of questions: " + str(count) +print "Average recall over questions: " + str(averageRecall) +print "Average precision over questions: " + str(averagePrecision) +print "Average f1 over questions: " + str(averageF1) +averageNewF1 = 2 * averageRecall * averagePrecision / (averagePrecision + averageRecall) +print "F1 of average recall and average precision: " + str(averageNewF1) + diff --git a/scripts/extract-module-classes.rb b/scripts/extract-module-classes.rb new file mode 100755 index 0000000..544fda0 --- /dev/null +++ b/scripts/extract-module-classes.rb @@ -0,0 +1,24 @@ +#!/usr/bin/ruby + +# Input: src +# Output: module-classes.txt + +# For each module (e.g., core, freebase), we compute the list of classes +# associated with that class. + +out_path = 'module-classes.txt' +out = open(out_path, 'w') +items = [] +core_packages = ['test'] # Packages in core which are not their own modules +modules = {} +Dir['src/**/*.java'].each { |path| + class_name = path.sub(/^src\//, '').gsub(/\//, '.').gsub(/\.java$/, '') + module_name = path.sub(/^.*sempre\//, '').split(/\//)[0] + module_name = 'core' if module_name =~ /\.java$/ || core_packages.index(module_name) + modules[module_name] = true + items << (module_name + " " + class_name) +} +items.sort! +out.puts items +out.close +puts "Wrote modules with #{items.size} files to #{out_path}: #{modules.keys.sort.join(' ')}" diff --git a/scripts/find-first-pred-diff.sh b/scripts/find-first-pred-diff.sh new file mode 100755 index 0000000..cec807c --- /dev/null +++ b/scripts/find-first-pred-diff.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Takes two log files and outputs the first line where they differ in terms +# of the example or the predictions + +egrep "Example: |Pred@| Item " ../state/execs/$1.exec/log > temp.1 +egrep "Example: |Pred@| Item " ../state/execs/$2.exec/log > temp.2 +cmp temp.1 temp.2 +#rm temp.1 +#rm temp.2 + diff --git a/scripts/find-hard-coded-paths.rb b/scripts/find-hard-coded-paths.rb new file mode 100755 index 0000000..1797f19 --- /dev/null +++ b/scripts/find-hard-coded-paths.rb @@ -0,0 +1,19 @@ +#!/usr/bin/ruby + +# Heuristically find all hard-coded paths in the source code. +# There should be no absolute paths. +Dir["src/**/*.java"].each { |sourcePath| + IO.foreach(sourcePath) { |line| + next unless line =~ /"([\w_\/\.]+)"/ + file = $1 + next unless file =~ /^\/[uU]\w+\/\w+/ || file =~ /^lib\// + if file =~ /^\// + message = " [BAD: absolute path]" + elsif not File.exists?(file) + message = " [BAD: does not exist]" + else + message = "" + end + puts sourcePath + ": " + file + message + } +} diff --git a/scripts/fix-checkstyle.rb b/scripts/fix-checkstyle.rb new file mode 100755 index 0000000..32f9e0b --- /dev/null +++ b/scripts/fix-checkstyle.rb @@ -0,0 +1,113 @@ +#!/usr/bin/ruby + +# Hacky script for automatically fixing style errors. This script is far from +# perfect and you should manually inspect all changes before making changes. + +# Usage: +# scripts/fix-checkstyle.rb # Inspect changes +# scripts/fix-checkstyle.rb mutate # Apply +# scripts/fix-checkstyle.rb # Do to particular files + +mutate = ARGV.index('mutate'); ARGV = ARGV.select { |a| a != 'mutate' } + +files = ARGV.size > 0 ? ARGV : Dir['src/**/*.java'] + +files.each { |path| + in_multi_comment = false + line_num = 0 + lines = IO.readlines(path).map { |line| + line_num += 1 + line = line.chomp + new_line = line + '' + + # Remove trailing whitespace + new_line.gsub!(/\s+$/, '') + + # Add space after comment + new_line.gsub!(/ \/\/(\w)/, '// \1') + + # Put space after certain operators + new_line.gsub!(/ (if|for|while|catch)\(/, ' \1 (') + + # Put space + new_line.gsub!(/( for \(\w+ \w+): /, '\1 : ') + + # Put space after casts + new_line.gsub!(/\((\w+)\)([a-z])/, '(\1) \2') + + # Reverse modifier order + new_line.gsub!(/ final static /, ' static final ') + + in_single_quote = false + in_double_quote = false + in_comment = false + tokens = new_line.split(//) + (0...tokens.size).each { |i| + # Previous, current, next characters + p = i-1 >= 0 ? tokens[i-1] : '' + c = tokens[i] + n = i+1 < tokens.size ? tokens[i+1] : '' + + if c == '\'' && p != '\\' # Quote + in_single_quote = !in_single_quote + end + if c == '"' && p != '\\' # Quote + in_double_quote = !in_double_quote + end + if c == '/' && n == '/' # Comment + in_comment = true + end + if c == '/' && n == '*' # Begin multi-comment + in_multi_comment = true + end + + if !(in_single_quote || in_double_quote || in_comment || in_multi_comment || c == '') + # Replace if not in quote or comment + if c == ',' # One space after + c += ' ' if n != ' ' + elsif ['++', '--'].index(c+n) # Double character operators + c = c+n + n = '' + elsif ['==', '!=', '<=', '>=', '+=', '-=', '*=', '/=', '&=', '|=', '&&', '||'].index(c+n) # Double character operators + c = c+n + n = '' + c = ' ' + c if p != ' ' + c = c + ' ' if tokens[i+2] != ' ' + elsif '*/=%'.index(c) # Single character operators + # Don't do <, > because of generics + # Don't do , + because of unaries + c = ' ' + c if p != ' ' + c = c + ' ' if n != ' ' + end + + # Write back + tokens[i] = c + tokens[i+1] = n if i+1 < tokens.size + end + + if p == '*' && c == '/' # End multi-comment + in_multi_comment = false + end + } + new_line = tokens.join('') + #p new_line + + new_line.gsub!(/\. \* ;/, '.*;') + new_line.gsub!(/\s+$/, '') + + if line != new_line + puts "======= #{path} #{line_num}" + puts "OLD: [#{line}]" + puts "NEW: [#{new_line}]" + end + + new_line + } + + # Write it out + if mutate + out = open(path, 'w') + out.puts lines + out.close + end +} diff --git a/scripts/tunnel b/scripts/tunnel new file mode 100755 index 0000000..8ed2aa8 --- /dev/null +++ b/scripts/tunnel @@ -0,0 +1,38 @@ +#!/bin/bash + +# Allows running things locally that need to connect to cache servers or sparql +# servers on NLP, which are behind a firewall. +# After starting this script, connect to localhost: to connect to :. + +# Cache server +host=jonsson +port=4000 +echo "Tunnel localhost:$port => $host:$port" +pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}') +if [ -n "$pid" ]; then kill $pid; fi +ssh -N -n -L $port:$host:$port jacob.stanford.edu & + +# Sparql server for Freebase +host=jonsson +port=3093 +echo "Tunnel localhost:$port => $host:$port" +pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}') +if [ -n "$pid" ]; then kill $pid; fi +ssh -N -n -L $port:$host:$port jacob.stanford.edu & + +# Sparql server for Paleo +host=jonsson +port=3021 +echo "Tunnel localhost:$port => $host:$port" +pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}') +if [ -n "$pid" ]; then kill $pid; fi +ssh -N -n -L $port:$host:$port jacob.stanford.edu & + +# Cache server (immutable) +host=john5 +port=4001 +echo "Tunnel localhost:$port => $host:$port" +pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}') +if [ -n "$pid" ]; then kill $pid; fi +ssh -N -n -L $port:$host:$port jacob.stanford.edu & + diff --git a/scripts/verify-code-loop.rb b/scripts/verify-code-loop.rb new file mode 100755 index 0000000..cae18b6 --- /dev/null +++ b/scripts/verify-code-loop.rb @@ -0,0 +1,107 @@ +#!/usr/bin/ruby + +# Verifies that the codebase is sane (compiles, doesn't crash, gets reasonable +# accuracy) every once in a while. If something fails, an email is sent out + +$mode = ARGV[0] +if $mode != 'now' && $mode != 'loop' + puts "Usage: #{$0} (now|loop)" + puts " now: check immediately and exit (don't email)" + puts " loop: check only when stuff changes and loop (email if something breaks)" + exit 1 +end + +# Who should be notified if the code breaks. +$recipient = 'stanford-sempre@googlegroups.com' +$logPath = "verify-code-loop.log" + +# Send out the log file to all the recipients. +def emailLog(subject) + return unless $mode == 'loop' + + maxLines = 100 # Maximum number of lines to send via email. + numLines = IO.readlines($logPath).size + if numLines <= maxLines + command = "cat #{$logPath}" + else + # Take first few lines and last few lines to keep under maxLines. + command = "(head -#{maxLines/2} #{$logPath}; echo '... (#{numLines - maxLines} lines omitted) ...'; tail -#{maxLines/2} #{$logPath})" + end + command = "#{command} | mail -s '#{subject}' #{$recipient}" + puts "Emailing log file: #{command}" + system command or exit 1 +end + +def emailBroken + emailLog('sempre code is broken!') +end + +# Print to stdout and log file. +def log(line, newline=true) + line = "[#{`date`.chomp}] #{line}" + if newline + puts line + else + print line + end + out = open($logPath, 'a') + out.puts line + out.close +end + +# Run and command; if fail, send email. +def run(command, verbose) + log("======== Running: #{command}", false) if verbose + ok = system "#{command} >> #{$logPath} 2>&1" + puts " [#{ok ? 'ok' : 'failed'}]" if verbose + emailBroken if not ok + ok +end + +def restart + exit 1 if $mode == 'now' + + system "cat #{$logPath}" + + # In case there are updates + log("Restarting #{$0}...") + exec($0 + ' loop') +end + +log("Started verify-code loop version 2") +log("Writing to #{$logPath}...") +firstTime = true +while true + break if $mode == 'now' && (not firstTime) + + # Whenever there's a change to the repository, run a test + system "rm -f #{$logPath}" + if not run('git pull', false) + log("git pull failed - this is bad, let's just quit.") + break + end + + if $mode == 'loop' && system("grep -q 'Already up-to-date' #{$logPath}") + # No changes, just wait + sleep 60 + next + end + firstTime = false + + # Check everything + log("Testing...") + run('git log -3', true) or restart # Print out last commit messages + run('./pull-dependencies', true) or restart + run('make clean', true) or restart + run('make', true) or restart + + run('scripts/find-hard-coded-paths.rb', true) or restart + + # Run tests + run("./run @mode=test", true) or restart + + emailLog('sempre code passes tests!') + + break if $mode == 'now' + restart +end diff --git a/sempre b/sempre deleted file mode 100755 index dd838dd..0000000 --- a/sempre +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/ruby - -$: << 'fig/lib' -require 'execrunner' - -# Note: run this on the NLP machines because SPARQL server is running on jack. -if ARGV.size == 0 - puts "Usage:" - puts " ./sempre @mode=train [additional options] # Train semantic parser" - puts " ./sempre @mode=interact [additional options] # Interactive mode for testing" - puts - puts " ./sempre @mode=cacheserver [additional options] # Start cache server" - puts " ./sempre @mode=sparqlserver [additional options] # Start sparql server" - puts - puts " ./sempre @mode=filterfreebase [additional options] # Filter raw Freebase RDF dump" - puts " ./sempre @mode=indexfreebase [additional options] # Build Freebase index" - puts - puts " ./sempre @mode=test [additional options] # Run TestNG test suite" - puts - puts "Additional options can be any of the following:" - puts " - Additional program options (e.g., -BeamParser.beamSize 3)" - puts " - Execrunner options, which select the options to include (@data=0)" - exit 1 -end - -system "mkdir -p state/execs" -system "touch state/lastExec" - -def header - l( - letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')), - 'fig/bin/qcreate', - (File.exists?('/u/nlp/bin/java7') ? '/u/nlp/bin/java7' : 'java'), - '-ea', - '-Xmx15g', - '-cp', 'classes:'+Dir['lib/*.jar'].join(':'), - letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'), - nil) -end - -def testHeader - l( - 'java', - '-ea', - '-Xmx12g', - '-cp', 'classes:'+Dir['lib/*.jar'].join(':'), - 'org.testng.TestNG', - nil) -end - -$lexdir=6 - -def sparqlOpts - l( - required(:sparqlserver, 'host:port of the Sparql server'), - o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}), - nil) -end - -$defaultFeatureDomains = [ - 'basicStats', - 'alignmentScores', - 'tokensDistance', - 'context', - 'skipPos', - 'joinPos', - #'bridgeBinaryMatch', - 'wordSim', - #'lexAlign', - 'tokenMatch', - #'rule', - 'opCount', - 'constant', - 'denotation', - 'whType', - #'lemmaAndBinaries', -nil].compact - -def defaultOpts - l( - o('execDir', '_OUTPATH_'), o('overwriteExecDir'), - o('addToView', 0), - - selo(0, 'LexiconFn.lexiconClassName', - 'edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon', - 'edu.stanford.nlp.sempre.fbalignment.lexicons.WordNetExpansionLexicon', - 'edu.stanford.nlp.sempre.fbalignment.lexicons.GraphPropLexicon'), - - letDefault(:executor, 1), - sel(:executor, - l(), - sparqlOpts, - nil), - selo(:executor, 'Builder.executor', - 'FormulaMatchExecutor', # Measure accuracy using logical forms (faster); good proxy - 'SparqlExecutor', # Measure accuracy using answers (slower); executes SPARQL query on database - nil), - - letDefault(:binarylex,0), - sel(:binarylex, - l( - o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/'+$lexdir.to_s+'/binaryInfoStringAndAlignment.txt'), - o('BinaryLexicon.keyToSortBy','Intersection_size_typed'), - nil), - l( - sel(nil, - o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/graphprop/1/jaccardLexicon','lib/fb_data/graphprop/1/graphpropLexicon'), - o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/graphprop/2/jaccardLexicon','lib/fb_data/graphprop/2/graphpropLexicon'), - nil), - o('BinaryLexicon.keyToSortBy','graphprop_estimate'), - nil), - l( - o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/graphprop/1/jaccardLexicon'), - o('BinaryLexicon.keyToSortBy','jaccard'), - nil), - nil), - - # binary - o('BinaryLexicon.useOnlyJaccard',false), - - # unary - o('UnaryLexicon.unaryLexiconFilePath','lib/fb_data/'+$lexdir.to_s+'/unaryInfoStringAndAlignment.txt'), - - nil) -end - -def unbalancedTrainDevSplit - l(o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2)) -end -def balancedTrainDevSplit - l(o('Dataset.trainFrac', 0.5), o('Dataset.devFrac', 0.5)) -end - -def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath) - l( - required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or : (hit the cacheserver)'), - lambda { |e| - cacheserver = e[:cacheserver] - case cacheserver - when 'none' then l() - when 'local' then l( # Use files directly - don't run more than one job that does this! - o('Lexicon.cachePath', 'LexiconFn.cache'), - o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'), - nil) - else l( - o('Lexicon.cachePath', cacheserver + ':' + lexiconFnCachePath), - o('SparqlExecutor.cachePath', cacheserver + ':' + sparqlExecutorCachePath), - nil) - end - }, - nil) -end - -# tag is either "free917" or "webquestions" -def emnlp2013AblationExperiments(tag) - l( - letDefault(:ablation, 0), - # Ablation experiments (EMNLP) - sel(:ablation, - l(), # (0) Just run things normally - selo(nil, 'BeamParser.beamSize', 10, 50, 200), # (1) Vary beam size - selo(nil, 'Dataset.trainFrac', 0.1, 0.2, 0.4, 0.6), # (2) Vary training set size - sel(nil, # (3) Structural: only do join or only do bridge - o('Grammar.tags', l(tag, 'join')), - o('Grammar.tags', l(tag, 'bridge')), - nil), - sel(nil, # (4) Features - o('FeatureExtractor.featureDomains', *($defaultFeatureDomains+['lexAlign'])), # +lexAlign - o('FeatureExtractor.featureDomains', *($defaultFeatureDomains+['lexAlign']-['alignmentScores'])), # +lexAlign -alignmentScores - o('FeatureExtractor.featureDomains', *($defaultFeatureDomains-['denotation'])), # -denotation - o('FeatureExtractor.featureDomains', *($defaultFeatureDomains-['skipPos', 'joinPos'])), # -syntax features (skipPos, joinPos) - nil), - nil), - - letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1, 2, 3), - nil) -end - -def free917 - l( # Data - letDefault(:data, 0), - sel(:data, - l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2 - l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json')), # (1) Careful, this is test set! - nil), - - # Grammar - o('Grammar.inPaths', 'data/emnlp2013.inexactent.grammar'), - o('Grammar.tags', 'free917', 'bridge', 'join'), - - # Features - o('FeatureExtractor.featureDomains', *$defaultFeatureDomains), - - emnlp2013AblationExperiments('free917'), - - l( - o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'), - o('Lexicon.entitySearchStrategy','exact'), - cachePaths('LexiconFn.cache', 'SparqlExecutor.cache'), - nil), - nil) -end - -def webquestions - l( - # Data - letDefault(:data, 0), - sel(:data, - l( # Webquestions (dev) - o('Dataset.inPaths', - 'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'), - unbalancedTrainDevSplit, - nil), - l( # Webquestions (test) - o('Dataset.inPaths', - 'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json', - 'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'), - nil), - nil), - - # Grammar - o('Grammar.inPaths', 'data/emnlp2013.inexactent.grammar'), - o('Grammar.tags', 'webquestions', 'bridge', 'join'), - - o('BeamParser.beamSize', 200), - - # Features - o('FeatureExtractor.featureDomains', *($defaultFeatureDomains+['lexAlign'])), - - emnlp2013AblationExperiments('webquestions'), - - # Caching - cachePaths('LexiconFn.cache', 'SparqlExecutor.cache'), - - o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'), - o('EntityLexicon.maxEntries',50), - - # Learner reward - o('Learner.partialReward', true), - nil) -end - -def jayant - l( - # Data - letDefault(:data, 1), - sel(:data, - l(o('Dataset.inPaths', 'train,data/jayant-emnlp2012-validation.examples.json'), balancedTrainDevSplit), - l(o('Dataset.inPaths', 'train,data/jayant-emnlp2012-validation.examples.json','test,data/jayant-emnlp2012-test.examples.json')), - nil), - - # Grammar - o('Grammar.inPaths', 'data/emnlp2013.inexactent.grammar'), - o('Grammar.tags', 'webquestions', 'bridge', 'join'), - - o('BeamParser.beamSize', 500), - - # Caching - cachePaths('LexiconFn.cache', 'SparqlExecutor.cache'), - - o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'), - nil) -end - -def selectDomain - l( - letDefault(:domain, 0), - sel(:domain, { - 'none' => l(), - 'webquestions' => webquestions, - 'free917' => free917, - 'jayant' => jayant, - }), - nil) -end - -def train - l( - header, - 'edu.stanford.nlp.sempre.Main', - defaultOpts, selectDomain, - o('Learner.maxTrainIters', 4), - nil) -end - -def interact - l( - # After training, run interact, which loads up a set of parameters and - # puts you in a prompt. - system('which rlwrap') ? 'rlwrap' : nil, - header, - 'edu.stanford.nlp.sempre.Main', - defaultOpts, selectDomain, - o('Dataset.inPaths'), - o('Learner.maxTrainIters', 0), - required(:load, 'none or exec number (e.g., 812) to load'), - lambda { |e| - if e[:load] == 'none' then - l() - else - execPath = "lib/models/#{e[:load]}.exec" - l( - o('Builder.inParamsPath', execPath+'/params'), - o('Grammar.inPaths', execPath+'/grammar'), - o('Master.logPath', lambda{|e| 'state/' + e[:domain] + '.log'}), - o('Master.newExamplesPath', lambda{|e| 'state/' + e[:domain] + '.examples'}), - o('Master.onlineLearnExamples', true), - # Make sure features are set properly! - nil) - end - }, - o('Main.interactive'), - sel(:executeTopOnly, - l(), - l( - o('Parser.executeTopFormulaOnly'), - a('FeatureExtractor.disableDenotationFeatures', true), - nil), - nil), - nil) -end - -# Start the cache server that serves files with key-value maps. -def cacheserver - l( - 'java', '-Xmx15g', '-ea', '-cp', 'classes:lib/fig.jar', - 'edu.stanford.nlp.sempre.StringCacheServer', - o('port', 4000), - nil) -end - -############################################################ -# Freebase RDF database - -# Scratch directory -def scrOptions - letDefault(:scr, '/u/nlp/data/semparse/rdf/scr/' + `hostname | cut -f 1 -d .`.chomp) -end - -# (1) Filter RDF Freebase dump (do this once) [takes about 1 hour] -def filterfreebase - l( - scrOptions, - l( - 'fig/bin/qcreate', o('statePath', lambda{|e| e[:scr] + '/state'}), - 'java', '-ea', '-Xmx20g', '-cp', 'classes:lib/*', - 'edu.stanford.nlp.sempre.freebase.FilterFreebase', - o('inPath', '/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized'), - sel(:keep, { - 'all' => o('keepAllProperties'), - 'geo' => l( - o('keepTypesPaths', 'data/geo.types'), - o('keepPropertiesPath', 'data/geo.properties'), - o('keepGeneralPropertiesOnlyForSeenEntities', true), - nil), - }), - o('execDir', '_OUTPATH_'), o('overwriteExecDir'), - nil), - nil) -end - -# (2) Start the SPARQL server (do this every time). -def sparqlserver - l( - scrOptions, - letDefault(:exec, 93), - 'scripts/virtuoso', 'start', - lambda{|e| 'lib/freebase/'+e[:exec].to_s+'.exec/vdb'}, # DB directory - lambda{|e| 3000+e[:exec]}, # port - nil) -end - -# (3) Index the filtered RDF dump [takes 48 hours] -# Afterwards, stop the server and copy the exec directory to -# /u/nlp/data/semparse/scr/freebase/state/execs (this will serve as the master -# copy). -def indexfreebase - l( - letDefault(:stage, nil), - scrOptions, - required(:exec), - sel(:stage, - l( - 'scripts/virtuoso', 'add', - lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/0.ttl'}, # ttl file - lambda{|e| 3000+e[:exec]}, # port - lambda{|e| e[:offset] || 0}, # offset - nil), - l( - 'scripts/extract-freebase-schema.rb', - lambda{|e| 'http://localhost:'+(3000+e[:exec]).to_s+'/sparql'}, # port - lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/schema.ttl'}, - nil), - nil), - nil) -end - -# Query a single logical form or SPARQL -def query - l( - 'java', '-ea', - '-cp', 'classes:'+Dir['lib/*.jar'].join(':'), - 'edu.stanford.nlp.sempre.SparqlExecutor', - sparqlOpts, - nil) -end - -def test - l( - testHeader, - lambda { |e| - if e[:class] - l('-testclass', 'edu.stanford.nlp.sempre.test.' + e[:class]) - else - 'testng.xml' - end - }, - nil) -end - -run!( - sel(:mode, { - 'train' => train, - 'interact' => interact, - 'cacheserver' => cacheserver, - 'filterfreebase' => filterfreebase, - 'indexfreebase' => indexfreebase, - 'sparqlserver' => sparqlserver, - 'query' => query, - 'test' => test, - }), -nil) diff --git a/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java b/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java new file mode 100644 index 0000000..a076274 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java @@ -0,0 +1,203 @@ +package edu.stanford.nlp.sempre; + +import com.google.common.collect.Sets; +import fig.basic.LogInfo; +import fig.basic.MapUtils; +import fig.basic.StopWatchSet; + +import java.util.*; + +/** + * Contains methods for putting derivations on the chart and combining them + * to add new derivations to the agenda + * @author joberant + */ +abstract class AbstractReinforcementParserState extends ChartParserState { + + protected final ReinforcementParser parser; + protected final CoarseParser coarseParser; + protected CoarseParser.CoarseParserState coarseParserState; + protected static final double EPSILON = 10e-20; // used to break ties between agenda items + + public AbstractReinforcementParserState(ReinforcementParser parser, Params params, Example ex, boolean computeExpectedCounts) { + super(parser, params, ex, computeExpectedCounts); + this.parser = parser; + coarseParser = parser.coarseParser; + } + + protected abstract void addToAgenda(DerivationStream derivationStream); + + protected boolean coarseAllows(String cat, int start, int end) { + return coarseParserState == null || coarseParserState.coarseAllows(cat, start, end); + } + + //don't add to a cell in the chart that is fill + protected boolean addToBoundedChart(Derivation deriv) { + + List derivations = chart[deriv.start][deriv.end].get(deriv.cat); + totalGeneratedDerivs++; + if (Parser.opts.visualizeChartFilling) { + chartFillingList.add(new CatSpan(deriv.start, deriv.end, deriv.cat)); + } + if (derivations == null) { + chart[deriv.start][deriv.end].put(deriv.cat, + derivations = new ArrayList<>()); + } + if (derivations.size() < getBeamSize()) { + derivations.add(deriv); + Collections.sort(derivations, Derivation.derivScoreComparator); // todo - perhaps can be removed + return true; + } else { + return false; + } + } + + // for [start, end) we try to create [start, end + i) or [start - i, end) and add unary rules + protected void combineWithChartDerivations(Derivation deriv) { + expandDerivRightwards(deriv); + expandDerivLeftwards(deriv); + applyCatUnaryRules(deriv); + } + + private void expandDerivRightwards(Derivation leftChild) { + if (parser.verbose(6)) + LogInfo.begin_track("Expanding rightward"); + Map> rhsCategoriesToRules = parser.leftToRightSiblingMap.get(leftChild.cat); + if (rhsCategoriesToRules != null) { + for (int i = 1; leftChild.end + i <= numTokens; ++i) { + Set intersection = Sets.intersection(rhsCategoriesToRules.keySet(), chart[leftChild.end][leftChild.end + i].keySet()); + + for (String rhsCategory : intersection) { + List compatibleRules = rhsCategoriesToRules.get(rhsCategory); + List rightChildren = chart[leftChild.end][leftChild.end + i].get(rhsCategory); + generateParentDerivations(leftChild, rightChildren, true, compatibleRules); + } + } + // handle terminals + if (leftChild.end < numTokens) + handleTerminalExpansion(leftChild, false, rhsCategoriesToRules); + } + if (parser.verbose(6)) + LogInfo.end_track(); + } + + private void expandDerivLeftwards(Derivation rightChild) { + if (parser.verbose(5)) + LogInfo.begin_track("Expanding leftward"); + Map> lhsCategorisToRules = parser.rightToLeftSiblingMap.get(rightChild.cat); + if (lhsCategorisToRules != null) { + for (int i = 1; rightChild.start - i >= 0; ++i) { + Set intersection = Sets.intersection(lhsCategorisToRules.keySet(), chart[rightChild.start - i][rightChild.start].keySet()); + + for (String lhsCategory : intersection) { + List compatibleRules = lhsCategorisToRules.get(lhsCategory); + List leftChildren = chart[rightChild.start - i][rightChild.start].get(lhsCategory); + generateParentDerivations(rightChild, leftChildren, false, compatibleRules); + } + } + // handle terminals + if (rightChild.start > 0) + handleTerminalExpansion(rightChild, true, lhsCategorisToRules); + } + if (parser.verbose(5)) + LogInfo.end_track(); + } + + private void generateParentDerivations(Derivation expandedDeriv, List otherDerivs, + boolean expandedLeftChild, List compatibleRules) { + + for (Derivation otherDeriv : otherDerivs) { + Derivation leftChild, rightChild; + if (expandedLeftChild) { + leftChild = expandedDeriv; + rightChild = otherDeriv; + } else { + leftChild = otherDeriv; + rightChild = expandedDeriv; + } + List children = new ArrayList<>(); + children.add(leftChild); + children.add(rightChild); + for (Rule rule : compatibleRules) { + if (coarseAllows(rule.lhs, leftChild.start, rightChild.end)) { + DerivationStream resDerivations = applyRule(leftChild.start, rightChild.end, rule, children); + + if (!resDerivations.hasNext()) + continue; + addToAgenda(resDerivations); + } + } + } + } + + // returns the score of derivation computed + private DerivationStream applyRule(int start, int end, Rule rule, List children) { + try { + if (Parser.opts.verbose >= 5) + LogInfo.logs("applyRule %s %s %s %s", start, end, rule, children); + StopWatchSet.begin(rule.getSemRepn()); // measuring time + StopWatchSet.begin(rule.toString()); + DerivationStream results = rule.sem.call(ex, + new SemanticFn.CallInfo(rule.lhs, start, end, rule, com.google.common.collect.ImmutableList.copyOf(children))); + StopWatchSet.end(); + StopWatchSet.end(); + return results; + } catch (Exception e) { + LogInfo.errors("Composition failed: rule = %s, children = %s", rule, children); + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private void applyCatUnaryRules(Derivation deriv) { + if (parser.verbose(4)) + LogInfo.begin_track("Category unary rules"); + for (Rule rule : parser.catUnaryRules) { + if (!coarseAllows(rule.lhs, deriv.start, deriv.end)) + continue; + if (deriv.cat.equals(rule.rhs.get(0))) { + DerivationStream resDerivations = applyRule(deriv.start, deriv.end, rule, Collections.singletonList(deriv)); + addToAgenda(resDerivations); + } + } + if (parser.verbose(4)) + LogInfo.end_track(); + } + + public List gatherRhsTerminalsDerivations() { + List derivs = new ArrayList<>(); + final List empty = Collections.emptyList(); + + for (int i = 0; i < numTokens; i++) { + for (int j = i + 1; j <= numTokens; j++) { + for (Rule rule : MapUtils.get(parser.terminalsToRulesList, phrases[i][j], Collections.emptyList())) { + if (!coarseAllows(rule.lhs, i, j)) + continue; + derivs.add(applyRule(i, j, rule, empty)); + } + } + } + return derivs; + } + + // rules where one word is a terminal and the other is a non-terminal + private void handleTerminalExpansion(Derivation child, boolean before, Map> categoriesToRules) { + + String phrase = before ? phrases[child.start - 1][child.start] : phrases[child.end][child.end + 1]; + int start = before ? child.start - 1 : child.start; + int end = before ? child.end : child.end + 1; + + if (categoriesToRules.containsKey(phrase)) { + List children = new ArrayList<>(); + children.add(child); + for (Rule rule : categoriesToRules.get(phrase)) { + if (coarseAllows(rule.lhs, start, end)) { + DerivationStream resDerivations = applyRule(start, end, rule, children); + if (!resDerivations.hasNext()) + continue; + addToAgenda(resDerivations); + } + } + } + } +} diff --git a/src/edu/stanford/nlp/sempre/AggregateFormula.java b/src/edu/stanford/nlp/sempre/AggregateFormula.java index 8e1df18..32b875b 100644 --- a/src/edu/stanford/nlp/sempre/AggregateFormula.java +++ b/src/edu/stanford/nlp/sempre/AggregateFormula.java @@ -3,14 +3,15 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** - * Aggregate takes a set and computes some number of that set. - * Includes existential quantification. + * 'Aggregate' takes a set and computes some number on that set. * * @author Percy Liang */ public class AggregateFormula extends Formula { - public enum Mode {count, sum, mean, min, max, exists}; + public enum Mode { count, sum, avg, min, max }; public final Mode mode; public final Formula child; @@ -29,10 +30,9 @@ public class AggregateFormula extends Formula { public static Mode parseMode(String mode) { if ("count".equals(mode)) return Mode.count; if ("sum".equals(mode)) return Mode.sum; - if ("mean".equals(mode)) return Mode.mean; + if ("avg".equals(mode)) return Mode.avg; if ("min".equals(mode)) return Mode.min; if ("max".equals(mode)) return Mode.max; - if ("exists".equals(mode)) return Mode.max; return null; } @@ -41,6 +41,14 @@ public class AggregateFormula extends Formula { return result == null ? new AggregateFormula(mode, child.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) + res.addAll(child.mapToList(func, alwaysRecurse)); + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof AggregateFormula)) return false; @@ -49,7 +57,7 @@ public class AggregateFormula extends Formula { if (!this.child.equals(that.child)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + mode.toString().hashCode(); diff --git a/src/edu/stanford/nlp/sempre/ArithmeticFormula.java b/src/edu/stanford/nlp/sempre/ArithmeticFormula.java new file mode 100644 index 0000000..9b28331 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ArithmeticFormula.java @@ -0,0 +1,85 @@ +package edu.stanford.nlp.sempre; + +import com.google.common.base.Function; +import fig.basic.LispTree; + +import java.util.List; + +/** + * Performs arithmetic operations (+, -, *, /). + * Note that these are non-binary relations, which means we can't model them + * using a join. + * + * @author Percy Liang + */ +public class ArithmeticFormula extends Formula { + public enum Mode { add, sub, mul, div }; + public final Mode mode; + public final Formula child1; + public final Formula child2; + + public ArithmeticFormula(Mode mode, Formula child1, Formula child2) { + this.mode = mode; + this.child1 = child1; + this.child2 = child2; + } + + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild(modeToString(mode)); + tree.addChild(child1.toLispTree()); + tree.addChild(child2.toLispTree()); + return tree; + } + + public Formula map(Function func) { + Formula result = func.apply(this); + return result == null ? new ArithmeticFormula(mode, child1.map(func), child2.map(func)) : result; + } + + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) { + res.addAll(child1.mapToList(func, alwaysRecurse)); + res.addAll(child2.mapToList(func, alwaysRecurse)); + } + return res; + } + + public static Mode parseMode(String mode) { + if ("+".equals(mode)) return Mode.add; + if ("-".equals(mode)) return Mode.sub; + if ("*".equals(mode)) return Mode.mul; + if ("/".equals(mode)) return Mode.div; + return null; + } + + public static String modeToString(Mode mode) { + switch (mode) { + case add: return "+"; + case sub: return "-"; + case mul: return "*"; + case div: return "/"; + default: throw new RuntimeException("Invalid mode: " + mode); + } + } + + @Override + public boolean equals(Object thatObj) { + if (!(thatObj instanceof ArithmeticFormula)) return false; + ArithmeticFormula that = (ArithmeticFormula) thatObj; + if (this.mode != that.mode) return false; + if (!this.child1.equals(that.child1)) return false; + if (!this.child2.equals(that.child2)) return false; + return true; + } + + public int computeHashCode() { + int hash = 0x7ed55d16; + hash = hash * 0xd3a2646c + mode.toString().hashCode(); // Note: don't call hashCode() on mode directly. + hash = hash * 0xd3a2646c + child1.hashCode(); + hash = hash * 0xd3a2646c + child2.hashCode(); + return hash; + } +} diff --git a/src/edu/stanford/nlp/sempre/AtomicSemType.java b/src/edu/stanford/nlp/sempre/AtomicSemType.java new file mode 100644 index 0000000..15a3bd2 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/AtomicSemType.java @@ -0,0 +1,31 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LispTree; +import java.util.*; + +// Represents an atomic type (strings, entities, numbers, dates, etc.). +public class AtomicSemType extends SemType { + public final String name; + public AtomicSemType(String name) { + if (name == null) throw new RuntimeException("Null name"); + this.name = name; + } + public boolean isValid() { return true; } + public SemType meet(SemType that) { + if (that instanceof TopSemType) return this; + if (that instanceof UnionSemType) return that.meet(this); + if (that instanceof AtomicSemType) { + String name1 = this.name; + String name2 = ((AtomicSemType) that).name; + if (name1.equals(name2)) return this; // Shortcut: the same + if (SemTypeHierarchy.singleton.getSupertypes(name1).contains(name2)) return this; + if (SemTypeHierarchy.singleton.getSupertypes(name2).contains(name1)) return that; + return SemType.bottomType; + } + return SemType.bottomType; + } + + public SemType apply(SemType that) { return SemType.bottomType; } + public SemType reverse() { return SemType.bottomType; } + public LispTree toLispTree() { return LispTree.proto.newLeaf(name); } +} diff --git a/src/edu/stanford/nlp/sempre/BadFormulaException.java b/src/edu/stanford/nlp/sempre/BadFormulaException.java new file mode 100644 index 0000000..cb4f6c8 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/BadFormulaException.java @@ -0,0 +1,21 @@ +package edu.stanford.nlp.sempre; + +public class BadFormulaException extends RuntimeException { + public static final long serialVersionUID = 86586128316354597L; + + String message; + + public BadFormulaException(String message) { this.message = message; } + + // Combine multiple exceptions + public BadFormulaException(BadFormulaException... exceptions) { + StringBuilder builder = new StringBuilder(); + for (BadFormulaException exception : exceptions) + builder.append(" | ").append(exception.message); + //builder.append(exception).append("\n"); + this.message = builder.toString().substring(3); + } + + @Override + public String toString() { return message; } +} diff --git a/src/edu/stanford/nlp/sempre/BeamParser.java b/src/edu/stanford/nlp/sempre/BeamParser.java index f369ad0..7f9bb16 100644 --- a/src/edu/stanford/nlp/sempre/BeamParser.java +++ b/src/edu/stanford/nlp/sempre/BeamParser.java @@ -4,6 +4,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import fig.basic.*; +import fig.exec.Execution; import java.util.*; @@ -11,46 +12,46 @@ import java.util.*; * A simple bottom-up chart-based parser that keeps the |beamSize| top * derivations for each chart cell (cat, start, end). Also supports fast * indexing of lexicalized rules using a trie. - *

- * In the future, when we have more parsers, some of this code should be - * refactored into Parser. + * + * Note that this code does not rely on the Grammar being binarized, + * which makes it more complex. * * @author Percy Liang */ public class BeamParser extends Parser { public static class Options { - @Option public int beamSize = 500; @Option public int maxNewTreesPerSpan = Integer.MAX_VALUE; } public static Options opts = new Options(); - Trie trie; //for non-catunary rules + Trie trie; // For non-cat-unary rules + + public BeamParser(Spec spec) { + super(spec); - public BeamParser(Grammar grammar, FeatureExtractor extractor, Executor executor) { - super(grammar, extractor, executor); // Index the non-cat-unary rules trie = new Trie(); for (Rule rule : grammar.rules) - if (!rule.isCatUnary()) - trie.add(rule); + addRule(rule); + if (Parser.opts.visualizeChartFilling) + this.chartFillOut = IOUtils.openOutAppendEasy(Execution.getFile("chartfill")); } - public int getDefaultBeamSize() { - return BeamParser.opts.beamSize; + public synchronized void addRule(Rule rule) { + if (!rule.isCatUnary()) + trie.add(rule); } - public ParserState newCoarseParserState(Params params, Example ex) { - return new BeamParserState( - ParserState.Mode.bool, - this, params, ex, null); - } - - public ParserState newParserState(Params params, - Example ex, - ParserState coarseState) { - return new BeamParserState( - ParserState.Mode.full, - this, params, ex, coarseState); + public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) { + BeamParserState coarseState = null; + if (Parser.opts.coarsePrune) { + LogInfo.begin_track("Parser.coarsePrune"); + coarseState = new BeamParserState(this, params, ex, computeExpectedCounts, BeamParserState.Mode.bool, null); + coarseState.infer(); + coarseState.keepTopDownReachable(); + LogInfo.end_track(); + } + return new BeamParserState(this, params, ex, computeExpectedCounts, BeamParserState.Mode.full, coarseState); } } @@ -61,115 +62,98 @@ public class BeamParser extends Parser { * @author Percy Liang * @author Roy Frostig */ -class BeamParserState extends ParserState { - private final BeamParser parser; +class BeamParserState extends ChartParserState { + public final Mode mode; + // Modes: + // 1) Bool: just check if cells (cat, start, end) are reachable (to prune chart) + // 2) Full: compute everything + public enum Mode { bool, full } - public BeamParserState(Mode mode, - BeamParser parser, - Params params, - Example ex, - ParserState coarseState) { - super(mode, parser, params, ex, coarseState); + private final BeamParser parser; + private final BeamParserState coarseState; // Used to prune + + public BeamParserState(BeamParser parser, Params params, Example ex, boolean computeExpectedCounts, + Mode mode, BeamParserState coarseState) { + super(parser, params, ex, computeExpectedCounts); this.parser = parser; + this.mode = mode; + this.coarseState = coarseState; + } + + public void infer() { + if (numTokens == 0) + return; + + if (parser.verbose(2)) LogInfo.begin_track("ParserState.infer"); + + // Base case + for (Derivation deriv : gatherTokenAndPhraseDerivations()) { + featurizeAndScoreDerivation(deriv); + addToChart(deriv); + } + + // Recursive case + for (int len = 1; len <= numTokens; len++) + for (int i = 0; i + len <= numTokens; i++) + build(i, i + len); + + if (parser.verbose(2)) LogInfo.end_track(); + + // Visualize + if (parser.chartFillOut != null && Parser.opts.visualizeChartFilling && this.mode != Mode.bool) { + parser.chartFillOut.println(Json.writeValueAsStringHard(new ChartFillingData(ex.id, chartFillingList, + ex.utterance, ex.numTokens()))); + parser.chartFillOut.flush(); + } + + setPredDerivations(); + + if (mode == Mode.full) { + // Compute gradient with respect to the predicted derivations + ensureExecuted(); + if (computeExpectedCounts) { + expectedCounts = new HashMap<>(); + ParserState.computeExpectedCounts(predDerivations, expectedCounts); + } + } } // Create all the derivations for the span [start, end). - @Override protected void build(int start, int end) { applyNonCatUnaryRules(start, end, start, parser.trie, new ArrayList(), new IntRef(0)); - Set cellsPruned = new HashSet(); + Set cellsPruned = new HashSet<>(); applyCatUnaryRules(start, end, cellsPruned); - for (Map.Entry> entry : getChart()[start][end].entrySet()) + for (Map.Entry> entry : chart[start][end].entrySet()) pruneCell(cellsPruned, entry.getKey(), start, end, entry.getValue()); } - private void pruneCell(Set cellsPruned, String cat, int start, int end, List derivations) { - String cell = cellString(cat, start, end); - if (cellsPruned.contains(cell)) return; - cellsPruned.add(cell); - - // Keep stats - if (derivations.size() > maxCellSize) { - maxCellSize = derivations.size(); - maxCellDescription = String.format("[%s %s]", cat, getExample().spanString(start, end)); - if (maxCellSize > 5000) - LogInfo.logs("BeamParser.pruneCell %s: %s entries", maxCellDescription, maxCellSize); - } - - // The extra code blocks in here that set |deriv.maxXBeamPosition| - // are there to track, over the course of parsing, the lowest - // position at which any of a derivation's constituents ever - // placed on any of the relevant beams. - - // Max beam position (before sorting) - for (int i = 0; i < derivations.size(); i++) { - Derivation deriv = derivations.get(i); - deriv.maxUnsortedBeamPosition = i; - if (deriv.children != null) { - for (Derivation child : deriv.children) - deriv.maxUnsortedBeamPosition = Math.max(deriv.maxUnsortedBeamPosition, child.maxUnsortedBeamPosition); - } - if (deriv.preSortBeamPosition == -1) { - // Need to be careful to only do this once since |pruneCell()| - // might be called several times for the same beam and the - // second time around we have already sorted once. - deriv.preSortBeamPosition = i; - } - } - - Derivation.sortByScore(derivations); - - // Max beam position (after sorting) - for (int i = 0; i < derivations.size(); i++) { - Derivation deriv = derivations.get(i); - deriv.maxBeamPosition = i; - if (deriv.children != null) { - for (Derivation child : deriv.children) - deriv.maxBeamPosition = Math.max(deriv.maxBeamPosition, child.maxBeamPosition); - } - deriv.postSortBeamPosition = i; - } - - // Keep only the top hypotheses - int beamSize = getBeamSize(); - while (derivations.size() > beamSize) { - derivations.remove(derivations.size() - 1); - fallOffBeam = true; - } - - // Reduce memory - if (derivations instanceof ArrayList) - ((ArrayList) derivations).trimToSize(); - } - - static String cellString(String cat, int start, int end) { + private static String cellString(String cat, int start, int end) { return cat + ":" + start + ":" + end; } // Return number of new derivations added private int applyRule(int start, int end, Rule rule, List children) { - if (Parser.opts.verbose >= 5) - LogInfo.logs("applyRule %s %s %s %s", start, end, rule, children); + if (Parser.opts.verbose >= 5) LogInfo.logs("applyRule %s %s %s %s", start, end, rule, children); try { - if (getMode() == Mode.full) { + if (mode == Mode.full) { StopWatchSet.begin(rule.getSemRepn()); - List results = rule.sem.call( - getExample(), + DerivationStream results = rule.sem.call(ex, new SemanticFn.CallInfo(rule.lhs, start, end, rule, ImmutableList.copyOf(children))); StopWatchSet.end(); - for (Derivation newDeriv : results) { + while (results.hasNext()) { + Derivation newDeriv = results.next(); featurizeAndScoreDerivation(newDeriv); addToChart(newDeriv); } - return results.size(); - } else if (getMode() == Mode.bool) { + return results.estimatedSize(); + } else if (mode == Mode.bool) { Derivation deriv = new Derivation.Builder() - .cat(rule.lhs).start(start).end(end).rule(rule) - .children(ImmutableList.copyOf(children)) - .formula(Formula.nullFormula) - .createDerivation(); + .cat(rule.lhs).start(start).end(end).rule(rule) + .children(ImmutableList.copyOf(children)) + .formula(Formula.nullFormula) + .createDerivation(); addToChart(deriv); return 1; } else { @@ -182,6 +166,14 @@ class BeamParserState extends ParserState { } } + // Don't prune the same cell more than once. + protected void pruneCell(Set cellsPruned, String cat, int start, int end, List derivations) { + String cell = cellString(cat, start, end); + if (cellsPruned.contains(cell)) return; + cellsPruned.add(cell); + pruneCell(cell, derivations); + } + // Apply all unary rules with RHS category. // Before applying each unary rule (rule.lhs -> rhsCat), we can prune the cell of rhsCat // because we assume acyclicity, so rhsCat's cell will never grow. @@ -190,7 +182,7 @@ class BeamParserState extends ParserState { if (!coarseAllows(rule.lhs, start, end)) continue; String rhsCat = rule.rhs.get(0); - List derivations = getChart()[start][end].get(rhsCat); + List derivations = chart[start][end].get(rhsCat); if (Parser.opts.verbose >= 5) LogInfo.logs("applyCatUnaryRules %s %s %s %s", start, end, rule, derivations); if (derivations == null) continue; @@ -228,7 +220,7 @@ class BeamParserState extends ParserState { // apply the rule on all the children gathered during the walk. if (i == end) { for (Rule rule : node.rules) { - if(coarseAllows(rule.lhs, start, end)) { + if (coarseAllows(rule.lhs, start, end)) { numNew.value += applyRule(start, end, rule, children); } } @@ -238,23 +230,86 @@ class BeamParserState extends ParserState { // Advance terminal token applyNonCatUnaryRules( start, end, i + 1, - node.next(getExample().token(i)), + node.next(ex.token(i)), children, numNew); // Advance non-terminal category for (int j = i + 1; j <= end; j++) { - for (Map.Entry> entry : getChart()[i][j].entrySet()) { + for (Map.Entry> entry : chart[i][j].entrySet()) { Trie nextNode = node.next(entry.getKey()); for (Derivation arg : entry.getValue()) { children.add(arg); applyNonCatUnaryRules(start, end, j, nextNode, children, numNew); children.remove(children.size() - 1); - if (getMode() != Mode.full) break; // Only need one hypothesis + if (mode != Mode.full) break; // Only need one hypothesis if (numNew.value >= BeamParser.opts.maxNewTreesPerSpan) return; } } } } -} + // -- Coarse state pruning -- + + // Remove any (cat, start, end) which isn't reachable from the + // (Rule.rootCat, 0, numTokens) + public void keepTopDownReachable() { + if (numTokens == 0) return; + + Set reachable = new HashSet<>(); + collectReachable(reachable, Rule.rootCat, 0, numTokens); + + // Remove all derivations associated with (cat, start, end) that aren't reachable. + for (int start = 0; start < numTokens; start++) { + for (int end = start + 1; end <= numTokens; end++) { + List toRemoveCats = new LinkedList<>(); + for (String cat : chart[start][end].keySet()) { + String key = catStartEndKey(cat, start, end); + if (!reachable.contains(key)) { + toRemoveCats.add(cat); + } + } + Collections.sort(toRemoveCats); + for (String cat : toRemoveCats) { + if (parser.verbose(4)) { + LogInfo.logs("Pruning chart %s(%s,%s)", cat, start, end); + } + chart[start][end].remove(cat); + } + } + } + } + + private void collectReachable(Set reachable, String cat, int start, int end) { + String key = catStartEndKey(cat, start, end); + if (reachable.contains(key)) return; + + if (!chart[start][end].containsKey(cat)) { + // This should only happen for the root when there are no parses. + return; + } + + reachable.add(key); + for (Derivation deriv : chart[start][end].get(cat)) { + for (Derivation subderiv : deriv.children) { + collectReachable(reachable, subderiv.cat, subderiv.start, subderiv.end); + } + } + } + + private String catStartEndKey(String cat, int start, int end) { + return cat + ":" + start + ":" + end; + } + + // For pruning with the coarse state + protected boolean coarseAllows(Trie node, int start, int end) { + if (coarseState == null) return true; + return SetUtils.intersects( + node.cats, + coarseState.chart[start][end].keySet()); + } + protected boolean coarseAllows(String cat, int start, int end) { + if (coarseState == null) return true; + return coarseState.chart[start][end].containsKey(cat); + } +} diff --git a/src/edu/stanford/nlp/sempre/BooleanValue.java b/src/edu/stanford/nlp/sempre/BooleanValue.java index c8fe360..ae478b8 100644 --- a/src/edu/stanford/nlp/sempre/BooleanValue.java +++ b/src/edu/stanford/nlp/sempre/BooleanValue.java @@ -15,14 +15,15 @@ public class BooleanValue extends Value { public LispTree toLispTree() { LispTree tree = LispTree.proto.newList(); tree.addChild("boolean"); - tree.addChild(value+""); + tree.addChild(value + ""); return tree; } @Override public int hashCode() { return Boolean.valueOf(value).hashCode(); } - @Override public boolean equals(Object thatObj) { - if (!(thatObj instanceof BooleanValue)) return false; - BooleanValue that = (BooleanValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BooleanValue that = (BooleanValue) o; return this.value == that.value; } } diff --git a/src/edu/stanford/nlp/sempre/BoundedPriorityQueue.java b/src/edu/stanford/nlp/sempre/BoundedPriorityQueue.java new file mode 100644 index 0000000..e2775c9 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/BoundedPriorityQueue.java @@ -0,0 +1,77 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; + +/** + * Created by joberant on 3/27/14. + * A priority queue that holds no more than N elements + */ +public class BoundedPriorityQueue extends TreeSet { + private static final long serialVersionUID = 5724671156522771658L; + private int elementsLeft; + + public BoundedPriorityQueue(int maxSize, Comparator comparator) { + super(comparator); + this.elementsLeft = maxSize; + } + + /** + * @return true if element was added, false otherwise + * */ + @Override + public boolean add(E e) { + if (elementsLeft == 0 && size() == 0) { + // max size was initiated to zero => just return false + return false; + } else if (elementsLeft > 0) { + // queue isn't full => add element and decrement elementsLeft + boolean added = super.add(e); + if (added) { + elementsLeft--; + } + return added; + } else { + // there is already 1 or more elements => compare to the least + int compared = super.comparator().compare(e, this.last()); + if (compared == -1) { + // new element is larger than the least in queue => pull the least and add new one to queue + pollLast(); + super.add(e); + return true; + } else { + // new element is less than the least in queue => return false + return false; + } + } + } + + public List toList() { + List res = new ArrayList<>(); + for (E e : this) + res.add(e); + return res; + } + + public static void main(String[] args) { + + BoundedPriorityQueue queue = + new BoundedPriorityQueue<>(5, + new Comparator() { + @Override + public int compare(Integer o1, Integer o2) { + return o1.compareTo(o2); + } + }); + + queue.add(10); + queue.add(8); + queue.add(4); + queue.add(12); + queue.add(3); + queue.add(7); + queue.add(9); + for (Integer num : queue) { + System.out.println(num); + } + } +} diff --git a/src/edu/stanford/nlp/sempre/BridgeFn.java b/src/edu/stanford/nlp/sempre/BridgeFn.java deleted file mode 100644 index b014aa3..0000000 --- a/src/edu/stanford/nlp/sempre/BridgeFn.java +++ /dev/null @@ -1,409 +0,0 @@ -package edu.stanford.nlp.sempre; - -import edu.stanford.nlp.sempre.FbFormulasInfo.BinaryFormulaInfo; -import edu.stanford.nlp.sempre.MergeFormula.Mode; -import fig.basic.LispTree; -import fig.basic.LogInfo; -import fig.basic.Option; -import fig.basic.IOUtils; - -import java.io.IOException; -import java.util.*; - -/** - * Bridge between two derivations by type-raising one of them. - * - * @author jonathanberant - */ -public class BridgeFn extends SemanticFn { - - private static final Formula intFormula = Formulas.fromLispTree(LispTree.proto.parseFromString("(fb:type.object.type fb:type.int)")); - private static final Formula floatFormula = Formulas.fromLispTree(LispTree.proto.parseFromString("(fb:type.object.type fb:type.float)")); - - public static class Options { - @Option(gloss = "Verbose") public int verbose = 0; - @Option(gloss = "Whether to allow entity bridging with no binary string match") public boolean looseEntBridge = false; - @Option(gloss = "List of binary formulas to use during bridging") public String binariesFile = ""; - } - - public static Options opts = new Options(); - - private FbFormulasInfo fbFormulaInfo = null; - private String description; - private boolean headFirst; - private TextToTextMatcher textToTextMatcher; - private static HashSet binariesToUse; - - public void init(LispTree tree) { - super.init(tree); - if (tree.children.size() != 3) - throw new RuntimeException("Number of children is: " + tree.children.size()); - if (!tree.child(2).value.equals("headFirst") && !tree.child(2).value.equals("headLast")) - throw new RuntimeException("Bad argument for head position: " + tree.child(2).value); - if (!tree.child(1).value.equals("unary") && !tree.child(1).value.equals("inject") && !tree.child(1).value.equals("entity")) - throw new RuntimeException("Bad description: " + tree.child(1).value); - - this.description = tree.child(1).value; - headFirst = tree.child(2).value.equals("headFirst"); - } - - public BridgeFn() { - fbFormulaInfo = FbFormulasInfo.getSingleton(); - textToTextMatcher = TextToTextMatcher.getSingleton(); - binariesToUse = new HashSet(); - if (!opts.binariesFile.equals("")) { - readBinaries(); - } - } - - public static void readBinaries() { - for (String line : IOUtils.readLinesHard(opts.binariesFile)) { - if (line.startsWith("#")) continue; - if (line.equals("")) continue; - binariesToUse.add(Formula.fromString(line)); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BridgeFn bridgeFn = (BridgeFn) o; - if (headFirst != bridgeFn.headFirst) return false; - if (!description.equals(bridgeFn.description)) return false; - return true; - } - - @Override - public List call(Example ex, Callable c) { - try { - if (description.equals("unary")) { - return bridgeUnary(ex, c); - } else if (description.equals("inject")) { - return injectIntoCvt(ex, c); - } else if (description.equals("entity")) { - return bridgeEntity(ex, c); - } else { - throw new RuntimeException("Invalid (expected unary, inject, or entity): " + description); - } - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(e); - } - } - - private boolean isCvt(Derivation headDeriv) { - if (!(headDeriv.formula instanceof JoinFormula)) - return false; - JoinFormula join = (JoinFormula) headDeriv.formula; - if (join.relation instanceof LambdaFormula) - return true; - if (join.child instanceof JoinFormula || join.child instanceof MergeFormula) - return true; - return false; - } - - // Return all the entity supertypes of |type|. - // TODO: make this more efficient. - private Set getSupertypes(SemType type, Set supertypes) { - if (type instanceof EntitySemType) - supertypes.addAll(fbFormulaInfo.getIncludedTypesInclusive(((EntitySemType) type).name)); - else if (type instanceof UnionSemType) - for (SemType baseType : ((UnionSemType) type).baseTypes) - getSupertypes(baseType, supertypes); - else { - // FIXME HACK for when passing binary into lambda formula and - // getSuperTypes doesn't work - getSupertypes(SemType.fromString("topic"), supertypes); - //throw new RuntimeException("Unexpected type (must be unary): " + type); - } - return supertypes; - } - - private List bridgeUnary(Example ex, Callable c) throws IOException { - - List res = new ArrayList(); - - // Example: modifier[Hanks] head[movies] - Derivation headDeriv = headFirst ? c.child(0) : c.child(1); - Derivation modifierDeriv = !headFirst ? c.child(0) : c.child(1); - - Set headTypes = getSupertypes(headDeriv.type, new HashSet()); - Set modifierTypes = getSupertypes(modifierDeriv.type, new HashSet()); - - HashSet usedBinaries = new HashSet(); - - for (String modifierType : modifierTypes) { // For each head type... - Set binaries; - if (binariesToUse.size() == 0) - binaries = fbFormulaInfo.getBinariesForType2(modifierType); - else - binaries = binariesToUse; - - for (Formula binary : binaries) { // For each possible binary... - - if (usedBinaries.contains(binary.toString())) - continue; - - BinaryFormulaInfo binaryInfo = fbFormulaInfo.getBinaryInfo(binary); - - if (opts.verbose >= 3) - LogInfo.logs("%s => %s", modifierType, binary); - - if (headTypes.contains(binaryInfo.expectedType1)) { - - Formula bridgedFormula = buildBridge(headDeriv.formula, modifierDeriv.formula, binary); - FbFormulasInfo.touchBinaryFormula(binary); - usedBinaries.add(binary.toString()); - - Derivation newDeriv = new Derivation.Builder() - .withCallable(c) - .formula(bridgedFormula) - .type(headDeriv.type) - .createDerivation(); - - if (SemanticFn.opts.trackLocalChoices) { - newDeriv.localChoices.add( - String.format( - "BridgeFn: %s %s --> %s %s --> %s %s", - headDeriv.startEndString(ex.getTokens()), headDeriv.formula, - ex.getTokens().subList(c.child(0).end, c.child(1).start), - binary, - modifierDeriv.startEndString(ex.getTokens()), modifierDeriv.formula)); - } - - // Add features - if (ex != null) { - newDeriv.addFeature("BridgeFn", "unary"); - - // Popularity of the binary - newDeriv.addFeatureWithBias("BridgeFn", "popularity", Math.log(binaryInfo.popularity + 1)); - - // head modifier POS tags - String headModifierOrder = headFirst ? "head-modifier" : "modifier-head"; - newDeriv.addFeature("BridgeFn", - "order=" + headModifierOrder + "," + - "pos=" + - ex.languageInfo.getCanonicalPos(headDeriv.start) + "-" + - ex.languageInfo.getCanonicalPos(modifierDeriv.start)); - - addBinaryMatchFeatures(ex, modifierDeriv, binary, newDeriv); //HACKY - List> exampleInfo = generateExampleInfo(ex, c); //this is not done in text to text matcher so done here - - FeatureVector vector = textToTextMatcher.extractFeatures( - exampleInfo.get(0), exampleInfo.get(1), exampleInfo.get(2), - new HashSet(binaryInfo.descriptions)); - newDeriv.addFeatures(vector); - } - res.add(newDeriv); - } - } - } - return res; - } - - //see whether the binary bridge can also be retrieved with the alignment lexicon - private void addBinaryMatchFeatures(Example ex, Derivation modifierDeriv, Formula binary, Derivation newDeriv) throws IOException { - if (!FeatureExtractor.containsDomain("bridgeBinaryMatch")) return; - - for (int i = 0; i < ex.languageInfo.lemmaTokens.size(); ++i) { - if (i >= modifierDeriv.start && i < modifierDeriv.end) - continue; - String pos = ex.languageInfo.posTags.get(i); - String lemma = ex.languageInfo.lemmaTokens.get(i); - if (pos.startsWith("NN") || (pos.startsWith("VB") && !pos.equals("VBD-AUX")) || pos.equals("JJ") || pos.equals("IN")) { - - LexiconFn fn = new LexiconFn(); - fn.init(LispTree.proto.parseFromString("(LexiconFn binary)")); - - Derivation child = new Derivation.Builder() - .cat("$CompositeRel").start(i).end(i + 1) - .children(new ArrayList()) - .withStringFormulaFrom(lemma) - .createDerivation(); - CallInfo c = new CallInfo("$Binary", i, i + 1, null, Collections.singletonList(child)); - List derivations = fn.call(ex, c); - for (Derivation deriv : derivations) { - if (deriv.formula.equals(binary) || deriv.formula.equals(FbFormulasInfo.reverseFormula(binary))) { - if (opts.verbose >= 1) { - LogInfo.logs("BridgeFn: lemma %s matched bridged binary %s", lemma, binary); - } - newDeriv.addFeatures(deriv); - return; - } - } - } - } - } - - //bridge without a unary - simply by looking at binaries leading to the entity and string matching binary description to example tokens/lemmas/stems - private List bridgeEntity(Example ex, Callable c) throws IOException { - - List res = new ArrayList(); - Derivation modifierDeriv = c.child(0); - - Set modifierTypes = getSupertypes(modifierDeriv.type, new HashSet()); - - for (String modifierType : modifierTypes) { // For each head type... - Set binaries = fbFormulaInfo.getBinariesForType2(modifierType); - for (Formula binary : binaries) { // For each possible binary... - BinaryFormulaInfo binaryInfo = fbFormulaInfo.getBinaryInfo(binary); - - if (opts.verbose >= 3) - LogInfo.logs("%s => %s", modifierType, binary); - - List> exampleInfo = generateExampleInfo(ex, c); //this is not done in text to text matcher so done here - if (textToTextMatcher.existsTokenMatch(exampleInfo.get(0), exampleInfo.get(2), new HashSet(binaryInfo.descriptions)) - || opts.looseEntBridge) { - Formula join = new JoinFormula(binary, modifierDeriv.formula); - FbFormulasInfo.touchBinaryFormula(binary); - - Derivation newDeriv = new Derivation.Builder() - .withCallable(c) - .formula(join) - .type(new EntitySemType(binaryInfo.expectedType1)) - .createDerivation(); - if (opts.verbose >= 2) - LogInfo.logs("BridgeStringFn: %s", join); - - //features - if (ex != null) { - newDeriv.addFeature("BridgeFn", "entity"); - newDeriv.addFeatureWithBias("BridgeFn", "popularity", Math.log(binaryInfo.popularity + 1)); - - addBinaryMatchFeatures(ex, modifierDeriv, binary, newDeriv); //HACKY - FeatureVector textMatchFeatures = textToTextMatcher.extractFeatures( - exampleInfo.get(0), exampleInfo.get(1), exampleInfo.get(2), - new HashSet(binaryInfo.descriptions)); - newDeriv.addFeatures(textMatchFeatures); - - //addTokenMatchFeatures(tokenStemFeatures.first(), newDeriv, "binary_token"); - //addTokenMatchFeatures(tokenStemFeatures.second(), newDeriv, "binary_stem"); - //addWordSimilarityFeatures(ex, newDeriv, binaryInfo); // (1) edit distance (2) word similarity - } - //newDeriv.localFeatureVector.add("bridge_lex_"+binaryInfo.expectedType1+"-->"+binary); //causes overfitting with 300 training examples - res.add(newDeriv); - } - } - } - return res; - } - - //generate from example array of content word tokens/lemmas/stems that are not dominated by child derivations - private List> generateExampleInfo(Example ex, Callable c) { - - List tokens = new ArrayList(); - List posTags = new ArrayList(); - List lemmas = new ArrayList(); - List> res = new ArrayList>(); - res.add(tokens); - res.add(posTags); - res.add(lemmas); - - Derivation modifierDeriv = headFirst ? c.child(1) : c.child(0); - - for (int i = 0; i < ex.languageInfo.tokens.size(); ++i) { - if (i >= modifierDeriv.start && i < modifierDeriv.end) { //do not consider the modifier words { - continue; - } - tokens.add(ex.languageInfo.tokens.get(i)); - posTags.add(ex.languageInfo.posTags.get(i)); - lemmas.add(ex.languageInfo.lemmaTokens.get(i)); - } - return res; - } - - private List injectIntoCvt(Example ex, Callable c) { - List res = new ArrayList(); - - // Example: modifier[Braveheart] head[Mel Gibson plays in] - Derivation headDeriv = headFirst ? c.child(0) : c.child(1); - if (!isCvt(headDeriv)) //only works on cvts - return res; - - Derivation modifierDeriv = !headFirst ? c.child(0) : c.child(1); - JoinFormula headFormula = (JoinFormula)Formulas.betaReduction(headDeriv.formula); - //find the type of the cvt node - Set headTypes = Collections.singleton(fbFormulaInfo.getBinaryInfo(headFormula.relation).expectedType2); - Set modifierTypes = getSupertypes(modifierDeriv.type, new HashSet()); - - for (String modifierType : modifierTypes) { - Set binaries = fbFormulaInfo.getAtomicBinariesForType2(modifierType); //here we use atomic binaries since we inject into a CVT - for (Formula binary : binaries) { // For each possible binary... - BinaryFormulaInfo info = fbFormulaInfo.getBinaryInfo(binary); - - if (headTypes.contains(info.expectedType1)) { - Formula bridgedFormula = buildBridgeFromCvt(headFormula, modifierDeriv.formula, binary); - FbFormulasInfo.touchBinaryFormula(binary); - Derivation newDeriv = new Derivation.Builder() - .withCallable(c) - .formula(bridgedFormula) - .type(headDeriv.type) - .createDerivation(); - if (opts.verbose >= 3) - LogInfo.logs("BridgeFn: injecting %s to %s --> %s ", modifierDeriv.formula, headFormula, bridgedFormula); - - if (ex != null) { - String headModifierOrder = headFirst ? "head-modifier" : "modifier-head"; - newDeriv.addFeature("BridgeFn", - "inject_order=" + headModifierOrder + "," + "pos=" + - ex.languageInfo.getCanonicalPos(headDeriv.start) + "-" + - ex.languageInfo.getCanonicalPos(modifierDeriv.start)); - } - newDeriv.addFeature("BridgeFn", "binary=" + binary); - - res.add(newDeriv); - } - } - } - return res; - } - - // Checks whether "var" is used as a binary in "formula" - private boolean varIsBinary(Formula formula, String var) { - boolean isBinary = false; - LispTree tree = formula.toLispTree(); - VariableFormula vf = new VariableFormula(var); - for (LispTree child : tree.children) { - if (child.isLeaf()) - continue; - if (child.children.size() == 2 && vf.equals(Formulas.fromLispTree(child.child(0)))) { - isBinary = true; - break; - } - if (varIsBinary(Formulas.fromLispTree(child), var)) { - isBinary = true; - break; - } - } - return isBinary; - } - - private Formula buildBridge(Formula headFormula, Formula modifierFormula, Formula binary) { - // Handle cases like "what state has the most cities" where "has the" is mapped - // to "contains" predicate via bridging but "most" triggers a nested lambda w/ - // argmax on a count relation - // (Corresponds to $MetaMetaOperator in grammar) - if (modifierFormula instanceof LambdaFormula) { - LambdaFormula lf = (LambdaFormula) modifierFormula; - if (varIsBinary(lf, lf.var)) { - Formula newBinary = Formulas.lambdaApply(lf, binary); - if (newBinary instanceof LambdaFormula) { - Formula result = Formulas.lambdaApply((LambdaFormula) newBinary, headFormula); - return result; - } - } - } - - Formula join = new JoinFormula(binary, modifierFormula); - Formula merge = new MergeFormula(Mode.and, headFormula, join); - //Don't merge on ints and floats - return (headFormula.equals(intFormula) || headFormula.equals(floatFormula)) ? join : merge; - } - - private Formula buildBridgeFromCvt(JoinFormula headFormula, Formula modifierFormula, Formula binary) { - Formula join = new JoinFormula(binary, modifierFormula); - Formula merge = new MergeFormula(Mode.and, headFormula.child, join); - return new JoinFormula(headFormula.relation, merge); - } -} diff --git a/src/edu/stanford/nlp/sempre/Builder.java b/src/edu/stanford/nlp/sempre/Builder.java index 8abdad9..2308713 100644 --- a/src/edu/stanford/nlp/sempre/Builder.java +++ b/src/edu/stanford/nlp/sempre/Builder.java @@ -1,6 +1,7 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Strings; + import fig.basic.Option; import fig.basic.Utils; @@ -12,9 +13,9 @@ import fig.basic.Utils; */ public class Builder { public static class Options { - @Option public String packageName = "edu.stanford.nlp.sempre"; @Option public String inParamsPath; - @Option public String executor = "SparqlExecutor"; + @Option public String executor = "JavaExecutor"; + @Option public String valueEvaluator = "ExactValueEvaluator"; @Option public String parser = "BeamParser"; } @@ -22,6 +23,7 @@ public class Builder { public Grammar grammar; public Executor executor; + public ValueEvaluator valueEvaluator; public FeatureExtractor extractor; public Parser parser; public Params params; @@ -29,6 +31,7 @@ public class Builder { public void build() { grammar = null; executor = null; + valueEvaluator = null; extractor = null; parser = null; params = null; @@ -45,19 +48,19 @@ public class Builder { // Executor if (executor == null) - executor = (Executor) Utils.newInstanceHard(opts.packageName + "." + opts.executor); + executor = (Executor) Utils.newInstanceHard(SempreUtils.resolveClassName(opts.executor)); - // Feature extractors + // Value evaluator + if (valueEvaluator == null) + valueEvaluator = (ValueEvaluator) Utils.newInstanceHard(SempreUtils.resolveClassName(opts.valueEvaluator)); + + // Feature extractor if (extractor == null) extractor = new FeatureExtractor(executor); // Parser - if (parser == null) { - if(opts.parser.equals("BeamParser")) - parser = new BeamParser(grammar, extractor, executor); - else - throw new RuntimeException("Illegal parser: " + opts.parser); - } + if (parser == null) + parser = buildParser(new Parser.Spec(grammar, extractor, executor, valueEvaluator)); // Parameters if (params == null) { @@ -66,4 +69,25 @@ public class Builder { params.read(opts.inParamsPath); } } + + public static Parser buildParser(Parser.Spec spec) { + switch (opts.parser) { + case "BeamParser": + return new BeamParser(spec); + case "ReinforcementParser": + return new ReinforcementParser(spec); + case "FloatingParser": + return new FloatingParser(spec); + default: + // Try instantiating by name + try { + Class parserClass = Class.forName(SempreUtils.resolveClassName(opts.parser)); + return (Parser) parserClass.getConstructor(spec.getClass()).newInstance(spec); + } catch (ClassNotFoundException e1) { + throw new RuntimeException("Illegal parser: " + opts.parser); + } catch (Exception e) { + throw new RuntimeException("Error while instantiating parser: " + opts.parser + "\n" + e); + } + } + } } diff --git a/src/edu/stanford/nlp/sempre/CallFormula.java b/src/edu/stanford/nlp/sempre/CallFormula.java index ebfeed2..5fba752 100644 --- a/src/edu/stanford/nlp/sempre/CallFormula.java +++ b/src/edu/stanford/nlp/sempre/CallFormula.java @@ -8,6 +8,7 @@ import java.util.List; /** * A CallFormula represents a function call. + * See JavaExecutor for the semantics of this formula. * (call func arg_1 ... arg_k) * * @author Percy Liang @@ -44,6 +45,17 @@ public class CallFormula extends Formula { return new CallFormula(newFunc, newArgs); } + @Override + public List mapToList(Function> transform, boolean alwaysRecurse) { + List res = transform.apply(this); + if (res.isEmpty() || alwaysRecurse) { + res.addAll(func.mapToList(transform, alwaysRecurse)); + for (Formula arg : args) + res.addAll(arg.mapToList(transform, alwaysRecurse)); + } + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof CallFormula)) return false; @@ -52,7 +64,7 @@ public class CallFormula extends Formula { if (!this.args.equals(that.args)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + func.hashCode(); diff --git a/src/edu/stanford/nlp/sempre/CallTypeInfo.java b/src/edu/stanford/nlp/sempre/CallTypeInfo.java new file mode 100644 index 0000000..841278c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/CallTypeInfo.java @@ -0,0 +1,16 @@ +package edu.stanford.nlp.sempre; + +import java.util.List; + +// Type information for each function in CallFormula. +public class CallTypeInfo { + public final String func; + public final List argTypes; + public final SemType retType; + + public CallTypeInfo(String func, List argTypes, SemType retType) { + this.func = func; + this.argTypes = argTypes; + this.retType = retType; + } +} diff --git a/src/edu/stanford/nlp/sempre/CanonicalNames.java b/src/edu/stanford/nlp/sempre/CanonicalNames.java new file mode 100644 index 0000000..87a3e03 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/CanonicalNames.java @@ -0,0 +1,42 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; + +/** + * List of canonical names that we borrowed from Freebase. + * + * These names and helper methods are independent from the Freebase schema. + * + * @author ppasupat + */ +public final class CanonicalNames { + private CanonicalNames() { } + + // Standard type names + public static final String BOOLEAN = "fb:type.boolean"; + public static final String INT = "fb:type.int"; + public static final String FLOAT = "fb:type.float"; + public static final String DATE = "fb:type.datetime"; + public static final String TEXT = "fb:type.text"; + public static final String NUMBER = "fb:type.number"; + public static final String ENTITY = "fb:common.topic"; + public static final String ANY = "fb:type.any"; + + public static final List PRIMITIVES = Collections.unmodifiableList( + Arrays.asList(BOOLEAN, INT, FLOAT, DATE, TEXT, NUMBER)); + + // Standard relations + public static final String TYPE = "fb:type.object.type"; + public static final String NAME = "fb:type.object.name"; + + // Return whether |property| is the name of a reverse property. + // Convention: ! is the prefix for reverses. + public static boolean isReverseProperty(String property) { + return property.startsWith("!") && !property.equals("!="); + } + public static String reverseProperty(String property) { + if (isReverseProperty(property)) return property.substring(1); + else return "!" + property; + } + +} diff --git a/src/edu/stanford/nlp/sempre/ChartParserState.java b/src/edu/stanford/nlp/sempre/ChartParserState.java new file mode 100644 index 0000000..6e8499d --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ChartParserState.java @@ -0,0 +1,122 @@ +package edu.stanford.nlp.sempre; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import fig.basic.LogInfo; +import fig.basic.MapUtils; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Actually does the parsing. Main method is infer(), whose job is to fill in + * + * @author Roy Frostig + * @author Percy Liang + */ +public abstract class ChartParserState extends ParserState { + // cell (start, end, category) -> list of derivations (sorted by decreasing score) [beam] + protected final Map>[][] chart; + + // For visualizing how chart is filled + List chartFillingList = new ArrayList<>(); + + protected String[][] phrases; // the phrases in the example + + @SuppressWarnings({ "unchecked" }) + public ChartParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) { + super(parser, params, ex, computeExpectedCounts); + + // Initialize the chart. + this.chart = (HashMap>[][]) + Array.newInstance(HashMap.class, numTokens, numTokens + 1); + this.phrases = new String[numTokens][numTokens + 1]; + + for (int start = 0; start < numTokens; start++) { + StringBuilder sb = new StringBuilder(); + for (int end = start + 1; end <= numTokens; end++) { + if (end - start > 1) + sb.append(' '); + sb.append(this.ex.languageInfo.tokens.get(end - 1)); + phrases[start][end] = sb.toString(); + chart[start][end] = new HashMap<>(); + } + } + } + + public void clearChart() { + for (int start = 0; start < numTokens; start++) { + for (int end = start + 1; end <= numTokens; end++) { + chart[start][end].clear(); + } + } + } + + // Call this method in infer() + protected void setPredDerivations() { + predDerivations.clear(); + predDerivations.addAll(MapUtils.get(chart[0][numTokens], Rule.rootCat, Derivation.emptyList)); + } + + private void visualizeChart() { + for (int len = 1; len <= numTokens; ++len) { + for (int i = 0; i + len <= numTokens; ++i) { + for (String cat : chart[i][i + len].keySet()) { + List derivations = chart[i][i + len].get(cat); + for (Derivation deriv : derivations) { + LogInfo.logs("ParserState.visualize: %s(%s:%s): %s", cat, i, i + len, deriv); + } + } + } + } + } + + void addToChart(Derivation deriv) { + if (parser.verbose(3)) LogInfo.logs("addToChart %s: %s", deriv.cat, deriv); + + List derivations = chart[deriv.start][deriv.end].get(deriv.cat); + if (chart[deriv.start][deriv.end].get(deriv.cat) == null) + chart[deriv.start][deriv.end].put(deriv.cat, derivations = new ArrayList<>()); + derivations.add(deriv); + totalGeneratedDerivs++; + + if (Parser.opts.visualizeChartFilling) { + chartFillingList.add(new CatSpan(deriv.start, deriv.end, deriv.cat)); + } + } + + // TODO(joberant): move to visualization utility class + public static class CatSpan { + @JsonProperty + public final int start; + @JsonProperty public final int end; + @JsonProperty public final String cat; + + @JsonCreator + public CatSpan(@JsonProperty("start") int start, @JsonProperty("end") int end, + @JsonProperty("cat") String cat) { + this.start = start; + this.end = end; + this.cat = cat; + } + } + + public static class ChartFillingData { + @JsonProperty public final String id; + @JsonProperty public final String utterance; + @JsonProperty public final int numOfTokens; + @JsonProperty public final List catSpans; + + @JsonCreator + public ChartFillingData(@JsonProperty("id") String id, @JsonProperty("catspans") List catSpans, + @JsonProperty("utterance") String utterance, @JsonProperty("numOfTokens") int numOfTokens) { + this.id = id; + this.utterance = utterance; + this.numOfTokens = numOfTokens; + this.catSpans = catSpans; + } + } +} diff --git a/src/edu/stanford/nlp/sempre/CoarseParser.java b/src/edu/stanford/nlp/sempre/CoarseParser.java new file mode 100644 index 0000000..908fcf7 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/CoarseParser.java @@ -0,0 +1,284 @@ +package edu.stanford.nlp.sempre; + +import com.google.common.base.Joiner; +import fig.basic.LogInfo; +import fig.basic.MapUtils; +import fig.basic.Pair; +import fig.basic.StopWatch; + +import java.lang.reflect.Array; +import java.util.*; + +/** + * Parser that only has information on what categories can parse what spans + * Does not hold backpointers for getting full parse, only reachability information + * Important: assumes that the grammar is binary + * Independent from the Parser code and therefore there is duplicate code (traverse(), keepTopDownReachable()) + * @author jonathanberant + */ +public class CoarseParser { + + public final Grammar grammar; + private Map, Set> rhsToLhsMap; + ArrayList catUnaryRules; // Unary rules with category on RHS + Map> terminalsToRulesList = new HashMap<>(); + + public CoarseParser(Grammar grammar) { + this.grammar = grammar; + catUnaryRules = new ArrayList<>(); + rhsToLhsMap = new HashMap<>(); + + Map> graph = new HashMap<>(); // Node from LHS to list of rules + for (Rule rule : grammar.rules) { + if (rule.rhs.size() > 2) + throw new RuntimeException("We assume that the grammar is binarized, rule: " + rule); + if (rule.isCatUnary()) + MapUtils.addToList(graph, rule.lhs, rule); + else if (rule.rhs.size() == 2) { // binary grammar + MapUtils.addToSet(rhsToLhsMap, Pair.newPair(rule.rhs.get(0), rule.rhs.get(1)), rule.lhs); + } else { + assert rule.isRhsTerminals(); + MapUtils.addToList(terminalsToRulesList, Joiner.on(' ').join(rule.rhs), rule); + } + } + // Topologically sort catUnaryRules so that B->C occurs before A->B + Map done = new HashMap<>(); + for (String node : graph.keySet()) + traverse(catUnaryRules, node, graph, done); + LogInfo.logs("Coarse parser: %d catUnaryRules (sorted), %d nonCatUnaryRules", catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size()); + } + + /** Helper function for transitive closure of unary rules. */ + private void traverse(List catUnaryRules, + String node, + Map> graph, + Map done) { + Boolean d = done.get(node); + if (Boolean.TRUE.equals(d)) return; + if (Boolean.FALSE.equals(d)) + throw new RuntimeException("Found cycle of unaries involving " + node); + done.put(node, false); + for (Rule rule : MapUtils.getList(graph, node)) { + traverse(catUnaryRules, rule.rhs.get(0), graph, done); + catUnaryRules.add(rule); + } + done.put(node, true); + } + + public CoarseParserState getCoarsePrunedChart(Example ex) { + CoarseParserState res = new CoarseParserState(ex, this); + res.infer(); + return res; + } + + class CoarseParserState { + + private Map>[][] chart; + public final Example example; + public final CoarseParser parser; + private int numTokens; + private long time; + private String[][] phrases; + + + + @SuppressWarnings({ "unchecked" }) + public CoarseParserState(Example example, CoarseParser parser) { + this.example = example; + this.parser = parser; + numTokens = example.numTokens(); + // Initialize the chart. + this.chart = (HashMap>[][]) + Array.newInstance( + HashMap.class, + numTokens, numTokens + 1); + phrases = new String[numTokens][numTokens + 1]; + + for (int start = 0; start < numTokens; start++) { + StringBuilder sb = new StringBuilder(); + for (int end = start + 1; end <= numTokens; end++) { + if (end - start > 1) + sb.append(' '); + sb.append(example.languageInfo.tokens.get(end - 1)); + phrases[start][end] = sb.toString(); + chart[start][end] = new HashMap<>(); + } + } + } + + public long getCoarseParseTime() { return time; } + + public void infer() { + + StopWatch watch = new StopWatch(); + watch.start(); + // parse with rules with tokens or RHS + parseTokensAndPhrases(); + // complete bottom up parsing + for (int len = 1; len <= numTokens; len++) + for (int i = 0; i + len <= numTokens; i++) + build(i, i + len); + // prune away things that are not reachable from the top + keepTopDownReachable(); + watch.stop(); + time = watch.getCurrTimeLong(); + } + + public boolean coarseAllows(String cat, int start, int end) { + return chart[start][end].containsKey(cat); + } + + private void build(int start, int end) { + handleBinaryRules(start, end); + handleUnaryRules(start, end); + } + + private void parseTokensAndPhrases() { + for (int i = 0; i < numTokens; ++i) { + addToChart(Rule.tokenCat, i, i + 1); + addToChart(Rule.lemmaTokenCat, i, i + 1); + } + for (int i = 0; i < numTokens; i++) { + for (int j = i + 1; j <= numTokens; j++) { + addToChart(Rule.phraseCat, i, j); + addToChart(Rule.lemmaPhraseCat, i, j); + } + } + } + + private void addToChart(String cat, int start, int end) { + if (Parser.opts.verbose >= 5) + LogInfo.logs("Adding to chart %s(%s,%s)", cat, start, end); + MapUtils.putIfAbsent(chart[start][end], cat, new ArrayList()); + } + + private void addToChart(String parentCat, String childCat, int start, int end) { + if (Parser.opts.verbose >= 5) + LogInfo.logs("Adding to chart %s(%s,%s)-->%s(%s,%s)", parentCat, start, end, childCat, start, end); + MapUtils.addToList(chart[start][end], parentCat, new CategorySpan(childCat, start, end)); } + + private void addToChart(String parentCat, String leftCat, String rightCat, int start, int i, int end) { + if (Parser.opts.verbose >= 5) + LogInfo.logs("Adding to chart %s(%s,%s)-->%s(%s,%s) %s(%s,%s)", parentCat, start, end, leftCat, start, i, rightCat, i, end); + MapUtils.addToList(chart[start][end], parentCat, new CategorySpan(leftCat, start, i)); + MapUtils.addToList(chart[start][end], parentCat, new CategorySpan(rightCat, i, end)); + } + + private void handleBinaryRules(int start, int end) { + for (int i = start + 1; i < end; ++i) { + List left = new ArrayList<>(chart[start][i].keySet()); + List right = new ArrayList<>(chart[i][end].keySet()); + if (i - start == 1) left.add(phrases[start][i]); // handle single terminal + if (end - i == 1) right.add(phrases[i][end]); // handle single terminal + + for (String l : left) { + for (String r : right) { + Set parentCats = rhsToLhsMap.get(Pair.newPair(l, r)); + + if (parentCats != null) { + for (String parentCat : parentCats) { + addToChart(parentCat, l, r, start, i, end); + } + } + } + } + } + } + + private void handleUnaryRules(int start, int end) { + + // terminals on RHS + for (Rule rule : MapUtils.get(terminalsToRulesList, phrases[start][end], Collections.emptyList())) { + addToChart(rule.lhs, start, end); + } + // catUnaryRules + for (Rule rule : parser.catUnaryRules) { + String rhsCat = rule.rhs.get(0); + if (chart[start][end].containsKey(rhsCat)) { + addToChart(rule.lhs, rhsCat, start, end); + } + } + } + + public void keepTopDownReachable() { + if (numTokens == 0) return; + + Set reachable = new HashSet(); + collectReachable(reachable, new CategorySpan(Rule.rootCat, 0, numTokens)); + + // Remove all derivations associated with (cat, start, end) that aren't reachable. + for (int start = 0; start < numTokens; start++) { + for (int end = start + 1; end <= numTokens; end++) { + List toRemoveCats = new LinkedList(); + for (String cat : chart[start][end].keySet()) { + if (!reachable.contains(new CategorySpan(cat, start, end))) { + toRemoveCats.add(cat); + } + } + Collections.sort(toRemoveCats); + for (String cat : toRemoveCats) { + if (Parser.opts.verbose >= 5) + LogInfo.logs("Pruning chart %s(%s,%s)", cat, start, end); + chart[start][end].remove(cat); + } + } + } + } + + private void collectReachable(Set reachable, CategorySpan catSpan) { + if (reachable.contains(catSpan)) + return; + if (!chart[catSpan.start][catSpan.end].containsKey(catSpan.cat)) { + // This should only happen for the root when there are no parses. + return; + } + reachable.add(catSpan); + for (CategorySpan childCatSpan : chart[catSpan.start][catSpan.end].get(catSpan.cat)) { + collectReachable(reachable, childCatSpan); + } + } + } + + class CategorySpan { + public final String cat; + public final int start; + public final int end; + + public CategorySpan(String cat, int start, int end) { + this.cat = cat; + this.start = start; + this.end = end; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((cat == null) ? 0 : cat.hashCode()); + result = prime * result + end; + result = prime * result + start; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + CategorySpan other = (CategorySpan) obj; + if (cat == null) { + if (other.cat != null) + return false; + } else if (!cat.equals(other.cat)) + return false; + if (end != other.end) + return false; + if (start != other.start) + return false; + return true; + } + } +} diff --git a/src/edu/stanford/nlp/sempre/Colorizer.java b/src/edu/stanford/nlp/sempre/Colorizer.java new file mode 100644 index 0000000..62291c2 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/Colorizer.java @@ -0,0 +1,50 @@ +package edu.stanford.nlp.sempre; + +/** + * Tools for colorizing output to console so easier to read + * + * @author Ziang Xie + */ + +public class Colorizer { + + public Colorizer() { } + + public String colorize(String s, String color) { + String cp = ""; + + // NOTE JDK 7+ feature + switch (color) { + case "black": + cp = "\u001B[30m"; + break; + case "red": + cp = "\u001B[31m"; + break; + case "green": + cp = "\u001B[32m"; + break; + case "yellow": + cp = "\u001B[33m"; + break; + case "blue": + cp = "\u001B[34m"; + break; + case "purple": + cp = "\u001B[35m"; + break; + case "cyan": + cp = "\u001B[36m"; + break; + case "white": + cp = "\u001B[37m"; + break; + default: + throw new RuntimeException("Invalid color: " + color); + } + + if (cp.equals("")) + return s; + return cp + s + "\u001B[0m"; + } +} diff --git a/src/edu/stanford/nlp/sempre/ConcatFn.java b/src/edu/stanford/nlp/sempre/ConcatFn.java index f95e2f7..39202e5 100644 --- a/src/edu/stanford/nlp/sempre/ConcatFn.java +++ b/src/edu/stanford/nlp/sempre/ConcatFn.java @@ -1,10 +1,6 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; - -import java.util.Collections; -import java.util.List; - /** * Takes two strings and returns their concatenation. * @@ -13,30 +9,31 @@ import java.util.List; public class ConcatFn extends SemanticFn { String delim; + public ConcatFn() { } + + public ConcatFn(String delim) { + this.delim = delim; + } + public void init(LispTree tree) { super.init(tree); delim = tree.child(1).value; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConcatFn concatFn = (ConcatFn) o; - if (!delim.equals(concatFn.delim)) return false; - return true; - } - - public List call(Example ex, Callable c) { - StringBuilder out = new StringBuilder(); - for (int i = 0; i < c.getChildren().size(); i++) { - if (i > 0) out.append(delim); - out.append(c.childStringValue(i)); - } - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .withStringFormulaFrom(out.toString()) - .createDerivation()); + public DerivationStream call(Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < c.getChildren().size(); i++) { + if (i > 0) out.append(delim); + out.append(c.childStringValue(i)); + } + return new Derivation.Builder() + .withCallable(c) + .withStringFormulaFrom(out.toString()) + .createDerivation(); + } + }; } } diff --git a/src/edu/stanford/nlp/sempre/ConstantFn.java b/src/edu/stanford/nlp/sempre/ConstantFn.java index 42cfbb1..c719d7a 100644 --- a/src/edu/stanford/nlp/sempre/ConstantFn.java +++ b/src/edu/stanford/nlp/sempre/ConstantFn.java @@ -2,9 +2,6 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; -import java.util.Collections; -import java.util.List; - /** * Just returns a fixed logical formula. * @@ -26,43 +23,26 @@ public class ConstantFn extends SemanticFn { if (2 < tree.children.size()) this.type = SemType.fromLispTree(tree.child(2)); else { - this.type = crudeInferType(formula); + this.type = TypeInference.inferType(formula); } + if (!this.type.isValid()) + throw new RuntimeException("ConstantFn: " + formula + " does not type check"); } - private SemType crudeInferType(Formula formula) { - // Try to infer the type - if (formula instanceof ValueFormula) { - Value value = ((ValueFormula)formula).value; - if (value instanceof NumberValue) return SemType.numberType; - else if (value instanceof StringValue) return SemType.stringType; - else if (value instanceof DateValue) return SemType.dateType; - else if (value instanceof NameValue) return SemType.entityType; - } else if (formula instanceof LambdaFormula) { - return new FuncSemType(SemType.topType, crudeInferType(((LambdaFormula)formula).body)); - } - - throw new RuntimeException("Can't infer type of " + formula); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConstantFn that = (ConstantFn) o; - if (!formula.equals(that.formula)) return false; - if (!type.equals(that.type)) return false; - return true; - } - - public List call(Example ex, Callable c) { - Derivation deriv = new Derivation.Builder() - .withCallable(c) - .formula(formula) - .type(type) - .createDerivation(); - if (FeatureExtractor.containsDomain("constant")) - deriv.addFeature("constant", ex.phraseString(c.getStart(), c.getEnd()) + " --- " + formula.toString()); - return Collections.singletonList(deriv); + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + Derivation res = new Derivation.Builder() + .withCallable(c) + .formula(formula) + .type(type) + .createDerivation(); + // don't generate feature if it is not grounded to a string + if (FeatureExtractor.containsDomain("constant") && c.getStart() != -1) + res.addFeature("constant", ex.phraseString(c.getStart(), c.getEnd()) + " --- " + formula.toString()); + return res; + } + }; } } diff --git a/src/edu/stanford/nlp/sempre/ContextFn.java b/src/edu/stanford/nlp/sempre/ContextFn.java new file mode 100644 index 0000000..c8e5f11 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ContextFn.java @@ -0,0 +1,123 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; +import fig.basic.*; + +/** + * Produces predicates (like LexiconFn) but do it from the logical forms + * in the context (inspects the ContextValue of the example). + * + * Takes depth, restrictType, and forbiddenTypes arguments allowing you + * to specify the depth/size and type of (formula) subtrees that you want to + * extract from the context. + * + * ONLY USE WITH TYPES!! + * + * E.g., + * + * (rule $X (context) (ContextFn (depth 0) (type fb:type.any)) + * would extract any unary/entity. + * + * (rule $X (context) (ContextFn (depth 1) (type (-> fb:type.any + * fb:type.any)) (forbidden (-> fb:type.any fb:type.something)) would + * extract all binaries except those with arg1 of type fb:type.something + * + * @author William Hamilton + */ +// TODO(Will): Reintegrate useful functionality from old implementation. +public class ContextFn extends SemanticFn { + // the depth/size of subtrees to extract + private int depth; + // the type that you want to extract + private SemType restrictType = SemType.topType; + + // set of types to not extract (overrides restrictType). + // For example, if restrict type is very general (e.g., (-> type.any type.any)) + // and you don't want some specific subtype (e.g., (-> type.something type.any)) + // then you would say specify (forbidden (-> type.something type.any)) + // and all subtypes of (-> type.any type.any) would be permissible + // except the forbidden one(s). + private Set forbiddenTypes = new HashSet(); + + public void init(LispTree tree) { + super.init(tree); + for (int i = 1; i < tree.children.size(); i++) { + LispTree arg = tree.child(i); + if ("type".equals(arg.child(0).value)) { + restrictType = SemType.fromLispTree(arg.child(1)); + } else if ("depth".equals(arg.child(0).value)) { + depth = Integer.parseInt(arg.child(1).value); + } else if ("forbidden".equals(arg.child(0).value)) { + forbiddenTypes.add(SemType.fromLispTree(arg.child(1))); + } else { + throw new RuntimeException("Unknown argument: " + arg); + } + } + } + + public DerivationStream call(final Example ex, final Callable c) { + return new MultipleDerivationStream() { + int index = 0; + List formulas; + + public Derivation createDerivation() { + if (ex.context == null) return null; + + if (formulas == null) { + formulas = new ArrayList(); + for (int i = ex.context.exchanges.size() - 1; i >= 0; i--) { + ContextValue.Exchange e = ex.context.exchanges.get(i); + extractFormulas(e.formula.toLispTree()); + } + } + if (index >= formulas.size()) return null; + Formula formula = formulas.get(index++); + for (SemType forbiddenType : forbiddenTypes) { + if (TypeInference.inferType(formula).meet(forbiddenType).isValid()) + return null; + } + return new Derivation.Builder() + .withCallable(c) + .formula(formula) + .type(TypeInference.inferType(formula)) + .createDerivation(); + } + + private void addFormula(Formula formula) { + if (formulas.contains(formula)) + return; + formulas.add(formula); + } + + // Extract from the logical form. + private void extractFormulas(LispTree formula) { + if (correctDepth(formula, 0) && typeCheck(formula)) { + addFormula(Formulas.fromLispTree(formula)); + } + if (formula.isLeaf()) + return; + for (LispTree child : formula.children) + extractFormulas(child); + } + + private boolean correctDepth(LispTree formula, int currentLevel) { + if (formula.isLeaf()) { + return currentLevel == depth; + } else { + boolean isCorrect = true; + for (LispTree child : formula.children) + isCorrect = isCorrect && correctDepth(child, currentLevel + 1); + return isCorrect; + } + } + + private boolean typeCheck(LispTree treeFormula) { + Formula formula = Formulas.fromLispTree(treeFormula); + SemType type = TypeInference.inferType(formula); + type = restrictType.meet(type); + return type.isValid(); + } + + }; + } +} diff --git a/src/edu/stanford/nlp/sempre/ContextValue.java b/src/edu/stanford/nlp/sempre/ContextValue.java new file mode 100644 index 0000000..c4145fd --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ContextValue.java @@ -0,0 +1,144 @@ +package edu.stanford.nlp.sempre; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import fig.basic.LispTree; +import java.util.*; + +/** + * Represents the discourse context (time, place, history of exchanges). + * This is part of an Example and used by ContextFn. + * + * @author Percy Liang + */ +public class ContextValue extends Value { + // A single exchange between the user and the system + // Note: we are not storing the entire derivation right now. + public static class Exchange { + public final String utterance; + public final Formula formula; + public final Value value; + public Exchange(String utterance, Formula formula, Value value) { + this.utterance = utterance; + this.formula = formula; + this.value = value; + } + public Exchange(LispTree tree) { + utterance = tree.child(1).value; + formula = Formulas.fromLispTree(tree.child(2)); + value = Values.fromLispTree(tree.child(3)); + } + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild("exchange"); + tree.addChild(utterance); + tree.addChild(formula.toLispTree()); + tree.addChild(value.toLispTree()); + return tree; + } + @Override public String toString() { return toLispTree().toString(); } + } + + public final String user; + public final DateValue date; + public final List exchanges; // List of recent exchanges with the user + public final KnowledgeGraph graph; // Mini-knowledge graph that captures the context + + public ContextValue withDate(DateValue newDate) { + return new ContextValue(user, newDate, exchanges, graph); + } + + public ContextValue withNewExchange(List newExchanges) { + return new ContextValue(user, date, newExchanges, graph); + } + + public ContextValue withGraph(KnowledgeGraph newGraph) { + return new ContextValue(user, date, exchanges, newGraph); + } + + public ContextValue(String user, DateValue date, List exchanges, KnowledgeGraph graph) { + this.user = user; + this.date = date; + this.exchanges = exchanges; + this.graph = graph; + } + + public ContextValue(String user, DateValue date, List exchanges) { + this(user, date, exchanges, null); + } + + public ContextValue(KnowledgeGraph graph) { + this(null, null, null, graph); + } + + // Example: + // (context (user pliang) + // (date 2014 4 20) + // (exchange "when was chopin born" (!fb:people.person.date_of_birth fb:en.frederic_chopin) (date 1810 2 22)) + // (graph NaiveKnowledgeGraph ((string Obama) (string "born in") (string Hawaii)) ...)) + public ContextValue(LispTree tree) { + String user = null; + DateValue date = null; + KnowledgeGraph graph = null; + exchanges = new ArrayList(); + for (int i = 1; i < tree.children.size(); i++) { + String key = tree.child(i).child(0).value; + if (key.equals("user")) { + user = tree.child(i).child(1).value; + } else if (key.equals("date")) { + date = new DateValue(tree.child(i)); + } else if (key.equals("graph")) { + graph = KnowledgeGraph.fromLispTree(tree.child(i)); + } else if (key.equals("exchange")) { + exchanges.add(new Exchange(tree.child(i))); + } else { + throw new RuntimeException("Invalid: " + tree.child(i)); + } + } + this.user = user; + this.date = date; + this.graph = graph; + } + + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild("context"); + if (user != null) + tree.addChild(LispTree.proto.newList("user", user)); + if (date != null) + tree.addChild(date.toLispTree()); + if (graph != null) + tree.addChild(graph.toLispTree()); + for (Exchange e : exchanges) + tree.addChild(LispTree.proto.newList("exchange", e.toLispTree())); + return tree; + } + + @Override public int hashCode() { + int hash = 0x7ed55d16; + hash = hash * 0xd3a2646c + user.hashCode(); + hash = hash * 0xd3a2646c + date.hashCode(); + hash = hash * 0xd3a2646c + exchanges.hashCode(); + hash = hash * 0xd3a2646c + graph.hashCode(); + return hash; + } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContextValue that = (ContextValue) o; + if (!this.user.equals(that.user)) return false; + if (!this.date.equals(that.date)) return false; + if (!this.exchanges.equals(that.exchanges)) return false; + if (!this.graph.equals(that.graph)) return false; + return true; + } + + @JsonValue + public String toString() { return toLispTree().toString(); } + + @JsonCreator + public static ContextValue fromString(String str) { + return new ContextValue(LispTree.proto.parseFromString(str)); + } +} diff --git a/src/edu/stanford/nlp/sempre/Dataset.java b/src/edu/stanford/nlp/sempre/Dataset.java index f1785ca..69d170a 100644 --- a/src/edu/stanford/nlp/sempre/Dataset.java +++ b/src/edu/stanford/nlp/sempre/Dataset.java @@ -9,6 +9,7 @@ import fig.basic.*; import fig.exec.Execution; import fig.prob.SampleUtils; +import java.io.*; import java.util.*; /** @@ -35,9 +36,6 @@ public class Dataset { @Option(gloss = "Only keep examples which have at most this number of tokens") public int maxTokens = Integer.MAX_VALUE; - - @Option(gloss = "Read dataset in full lisptree format (otherwise JSON).") - public boolean readLispTreeFormat = false; } public static Options opts = new Options(); @@ -89,18 +87,25 @@ public class Dataset { } public void readFromPathPairs(List> pathPairs) { - if (opts.readLispTreeFormat) { - readLispTreeFromPathPairs(pathPairs); - return; + // Try to detect whether we need JSON. + for (Pair pathPair : pathPairs) { + if (pathPair.getSecond().endsWith(".json")) { + readJsonFromPathPairs(pathPairs); + return; + } } + readLispTreeFromPathPairs(pathPairs); + } + + private void readJsonFromPathPairs(List> pathPairs) { List groups = Lists.newArrayListWithCapacity(pathPairs.size()); for (Pair pathPair : pathPairs) { String group = pathPair.getFirst(); String path = pathPair.getSecond(); List examples = Json.readValueHard( IOUtils.openInHard(path), - new TypeReference>() {}); + new TypeReference>() { }); GroupInfo gi = new GroupInfo(group, examples); gi.path = path; groups.add(gi); @@ -123,7 +128,7 @@ public class Dataset { LogInfo.end_track(); } - + private void splitDevFromTrain() { // Split original training examples randomly into train and dev. List origTrainExamples = allExamples.get("train"); @@ -143,8 +148,6 @@ public class Dataset { devExamples.add(origTrainExamples.get(perm[i])); } } - - private void readHelper(List incoming, int maxExamples, @@ -162,7 +165,7 @@ public class Dataset { ex = new Example.Builder().withExample(ex).setId(id).createExample(); } i++; - ex.preprocess(); + ex.preprocess(LanguageAnalyzer.getSingleton()); // Skip example if too long if (ex.numTokens() > opts.maxTokens) continue; @@ -176,27 +179,8 @@ public class Dataset { } } - private void collectStats() { - LogInfo.begin_track_printAll("Dataset stats"); - Execution.putLogRec("numTokenTypes", tokenTypes.size()); - Execution.putLogRec("numTokensPerExample", numTokensFig); - for (Map.Entry> e : allExamples.entrySet()) - Execution.putLogRec("numExamples." + e.getKey(), e.getValue().size()); - LogInfo.end_track(); - } - - /** - * For reading datasets entirely in lisptree format. - */ - @Deprecated - public void readLispTree() { - readLispTreeFromPathPairs(opts.inPaths); - } - - @Deprecated private void readLispTreeFromPathPairs(List> pathPairs) { LogInfo.begin_track_printAll("Dataset.read"); - for (Pair pathPair : pathPairs) { String group = pathPair.getFirst(); String path = pathPair.getSecond(); @@ -210,7 +194,6 @@ public class Dataset { LogInfo.end_track(); } - @Deprecated private void readLispTreeHelper(String path, int maxExamples, List examples) { if (examples.size() >= maxExamples) return; LogInfo.begin_track("Reading %s", path); @@ -225,7 +208,7 @@ public class Dataset { Example ex = Example.fromLispTree(tree, path + ":" + n); // Specify a default id if it doesn't exist n++; - ex.preprocess(); + ex.preprocess(LanguageAnalyzer.getSingleton()); // Skip example if too long if (ex.numTokens() > opts.maxTokens) continue; @@ -239,6 +222,15 @@ public class Dataset { LogInfo.end_track(); } + private void collectStats() { + LogInfo.begin_track_printAll("Dataset stats"); + Execution.putLogRec("numTokenTypes", tokenTypes.size()); + Execution.putLogRec("numTokensPerExample", numTokensFig); + for (Map.Entry> e : allExamples.entrySet()) + Execution.putLogRec("numExamples." + e.getKey(), e.getValue().size()); + LogInfo.end_track(); + } + private static int getMaxExamplesForGroup(String group) { int maxExamples = Integer.MAX_VALUE; for (Pair maxPair : opts.maxExamples) @@ -246,4 +238,19 @@ public class Dataset { maxExamples = maxPair.getSecond(); return maxExamples; } + + public static void appendExampleToFile(String path, Example ex) { + // JSON is an annoying format because we can't just append. + // So currently we have to read the entire file in and write it out. + List examples; + if (new File(path).exists()) { + examples = Json.readValueHard( + IOUtils.openInHard(path), + new TypeReference>() { }); + } else { + examples = new ArrayList(); + } + examples.add(ex); + Json.prettyWriteValueHard(new File(path), examples); + } } diff --git a/src/edu/stanford/nlp/sempre/DateFn.java b/src/edu/stanford/nlp/sempre/DateFn.java index 986b42a..1113dad 100644 --- a/src/edu/stanford/nlp/sempre/DateFn.java +++ b/src/edu/stanford/nlp/sempre/DateFn.java @@ -1,39 +1,27 @@ package edu.stanford.nlp.sempre; -import java.util.Collections; -import java.util.List; - /** * Maps a string to a Date. * * @author Percy Liang */ public class DateFn extends SemanticFn { - @Deprecated public static List dateAnnotations(String s) { - return Collections.emptyList(); - } - @Deprecated public static String dateAnnotation(String s) { - throw new RuntimeException("Remove me"); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - return o != null && getClass() == o.getClass(); - } - - public List call(Example ex, Callable c) { - String value = ex.languageInfo.getNormalizedNerSpan("DATE", c.getStart(), c.getEnd()); - if (value == null) return Collections.emptyList(); - - DateValue dateValue = DateValue.parseDateValue(value); - if (dateValue == null) return Collections.emptyList(); - - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .formula(new ValueFormula(dateValue)) - .type(SemType.dateType) - .createDerivation()); + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + String value = ex.languageInfo.getNormalizedNerSpan("DATE", c.getStart(), c.getEnd()); + if (value == null) + return null; + DateValue dateValue = DateValue.parseDateValue(value); + if (dateValue == null) + return null; + return new Derivation.Builder() + .withCallable(c) + .formula(new ValueFormula<>(dateValue)) + .type(SemType.dateType) + .createDerivation(); + } + }; } } diff --git a/src/edu/stanford/nlp/sempre/DateValue.java b/src/edu/stanford/nlp/sempre/DateValue.java index ddd195f..b3e1af7 100644 --- a/src/edu/stanford/nlp/sempre/DateValue.java +++ b/src/edu/stanford/nlp/sempre/DateValue.java @@ -1,6 +1,7 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; +import java.util.Calendar; public class DateValue extends Value { public final int year; @@ -19,7 +20,16 @@ public class DateValue extends Value { int year = -1, month = -1, day = -1; boolean isBC = dateStr.startsWith("-"); if (isBC) dateStr = dateStr.substring(1); - String dateParts[]; + + // Ignore time + int t = dateStr.indexOf('T'); + if (t != -1) dateStr = dateStr.substring(0, t); + + String[] dateParts; + + if (dateStr.indexOf('T') != -1) + dateStr = dateStr.substring(0, dateStr.indexOf('T')); + dateParts = dateStr.split("-"); if (dateParts.length > 3) throw new RuntimeException("Date has more than 3 parts: " + dateStr); @@ -41,6 +51,14 @@ public class DateValue extends Value { return val; } + public static DateValue now() { + Calendar cal = Calendar.getInstance(); + int year = cal.get(Calendar.YEAR); + int month = cal.get(Calendar.MONTH); + int day = cal.get(Calendar.DAY_OF_MONTH); + return new DateValue(year, month, day); + } + public DateValue(int year, int month, int day) { this.year = year; this.month = month; @@ -62,17 +80,6 @@ public class DateValue extends Value { return tree; } - public double getCompatibility(Value thatValue) { - if (!(thatValue instanceof DateValue)) - return 0; - DateValue that = (DateValue) thatValue; - // TODO: Only comparing the year right now. - boolean perfectMatch = (this.year == that.year); - //&& (this.month == that.month) - //&& (this.day == that.day); - return perfectMatch ? 1.0 : 0.0; - } - @Override public int hashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + year; @@ -81,9 +88,10 @@ public class DateValue extends Value { return hash; } - @Override public boolean equals(Object thatObj) { - if (!(thatObj instanceof DateValue)) return false; - DateValue that = (DateValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DateValue that = (DateValue) o; if (this.year != that.year) return false; if (this.month != that.month) return false; if (this.day != that.day) return false; diff --git a/src/edu/stanford/nlp/sempre/DerivOpCountFeatureComputer.java b/src/edu/stanford/nlp/sempre/DerivOpCountFeatureComputer.java new file mode 100644 index 0000000..ed1b110 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/DerivOpCountFeatureComputer.java @@ -0,0 +1,58 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; + +import com.google.common.collect.Sets; + +import fig.basic.*; + +/** + * Extracts indicator features that count how many times semantic functions and + * LHSs have been used in the derivation For now we count how many times MergeFn + * and JoinFn, also how many time unary, binary and entity lexical entries have + * been used. The feature is a pair with the operation and the count + * + * @author jonathanberant + */ +public class DerivOpCountFeatureComputer implements FeatureComputer { + public static class Options { + @Option(gloss = "Count only basic categories and SemanticFns") + public boolean countBasicOnly = true; + } + public static Options opts = new Options(); + + public static final String entityCat = "$Entity"; + public static final String unaryCat = "$Unary"; + public static final String binaryCat = "$Binary"; + public static final String joinFn = "JoinFn"; + public static final String mergeFn = "MergeFn"; + public static final String bridgeFn = "BridgeFn"; + public static Set featureNames = Sets.newHashSet(entityCat, unaryCat, binaryCat, joinFn, mergeFn, bridgeFn); + + @Override + public void extractLocal(Example ex, Derivation deriv) { + if (!FeatureExtractor.containsDomain("opCount")) return; + if (!deriv.isRoot(ex.numTokens())) return; + + // extract the operation count + Map opCounter = new HashMap<>(); + extractOperationsRecurse(deriv, opCounter); + addFeatures(deriv, opCounter); + } + + private void extractOperationsRecurse(Derivation deriv, Map opCounter) { + // Basic case: no rule + if (deriv.children.isEmpty()) return; + // increment counts for current rule + MapUtils.incr(opCounter, deriv.rule.lhs); + MapUtils.incr(opCounter, deriv.rule.sem.getClass().getSimpleName()); + // recursive call + for (Derivation child : deriv.children) + extractOperationsRecurse(child, opCounter); + } + + private void addFeatures(Derivation deriv, Map opCounter) { + for (String feature : (opts.countBasicOnly ? featureNames : opCounter.keySet())) + deriv.addFeature("opCount", "count(" + feature + ")=" + MapUtils.get(opCounter, feature, 0)); + } +} diff --git a/src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java b/src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java deleted file mode 100644 index c4adb5b..0000000 --- a/src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java +++ /dev/null @@ -1,81 +0,0 @@ -package edu.stanford.nlp.sempre; - -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.util.ArrayUtils; - -import java.util.Set; - -/** - * Extracts indicator features that count how many times semantic functions and - * LHSs have been used in the derivation For now we count how many times MergeFn - * and JoinFn, also how many time unary, binary and entity lexical entries have - * been used. The feature is a pair with the operation and the count - * - * @author jonathanberant - */ -public class DerivOpCountFeatureExtractor { - public static final String entityCat = "$Entity"; - public static final String unaryCat = "$Unary"; - public static final String binaryCat = "$Binary"; - public final static String joinFn = "JoinFn"; - public final static String mergeFn = "MergeFn"; - public final static String bridgeFn = "BridgeFn"; - public static Set featureNames = ArrayUtils.asSet(new String[]{entityCat, unaryCat, binaryCat, joinFn, mergeFn, bridgeFn}); - - public void extractLocal(Example ex, Derivation deriv) { - if (!FeatureExtractor.containsDomain("opCount")) return; - if (!deriv.isRoot(ex.numTokens())) return; - - //extract the operation count - Counter opCounter = new ClassicCounter(); - extractOperationsRecurse(deriv, opCounter); - addFeatures(deriv, opCounter); - - //add pre-terminal sequence - //StringBuilder sb = new StringBuilder(); - //extractPreterminalYieldRecurse(deriv, sb); - //deriv.localFeatureVector.add(sb.toString()); - } - - private void extractPreterminalYieldRecurse(Derivation deriv, StringBuilder sb) { - //base case 1 - if (deriv.children.size() == 0) - return; - //base case 2 - if (deriv.rule.lhs.equals(entityCat) || deriv.rule.lhs.equals(unaryCat) || deriv.rule.lhs.equals(binaryCat)) { - sb.append(deriv.rule.lhs + "_"); - return; - } - //recursive call from left to right - for (Derivation child : deriv.children) { - extractPreterminalYieldRecurse(child, sb); - } - } - - private void extractOperationsRecurse(Derivation deriv, Counter opCounter) { - - //base case - if (deriv.children.size() == 0) - return; - - //boolean[] appendPreTerminal = new boolean[deriv.children.size()]; - - //increment counts for current rule - opCounter.incrementCount(deriv.rule.lhs); - if (deriv.rule.sem instanceof JoinFn) - opCounter.incrementCount(joinFn); - else if (deriv.rule.sem instanceof MergeFn) - opCounter.incrementCount(mergeFn); - else if (deriv.rule.sem instanceof BridgeFn) - opCounter.incrementCount(bridgeFn); - //recursive call - for (Derivation child : deriv.children) - extractOperationsRecurse(child, opCounter); - } - - private void addFeatures(Derivation deriv, Counter opCounter) { - for (String feature : featureNames) - deriv.addFeature("opCount", "count(" + feature + ")=" + (int)opCounter.getCount(feature)); - } -} diff --git a/src/edu/stanford/nlp/sempre/Derivation.java b/src/edu/stanford/nlp/sempre/Derivation.java index fb1cfe6..783484b 100644 --- a/src/edu/stanford/nlp/sempre/Derivation.java +++ b/src/edu/stanford/nlp/sempre/Derivation.java @@ -1,8 +1,5 @@ package edu.stanford.nlp.sempre; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import fig.basic.*; import java.util.*; @@ -15,13 +12,18 @@ import java.util.*; * * @author Percy Liang */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public class Derivation implements SemanticFn.Callable { +public class Derivation implements SemanticFn.Callable, HasScore { public static class Options { @Option(gloss = "When printing derivations, to show values (could be quite verbose)") public boolean showValues = true; + @Option(gloss = "When printing derivations, to show the first value (ignored when showValues is set)") + public boolean showFirstValue = false; + @Option(gloss = "When printing derivations, to show types") + public boolean showTypes = true; @Option(gloss = "When printing derivations, to show rules") public boolean showRules = false; + @Option(gloss = "When printing derivations, to show canonical utterance") + public boolean showUtterance = false; } public static Options opts = new Options(); @@ -29,14 +31,13 @@ public class Derivation implements SemanticFn.Callable { //// Basic fields: created by the constructor. // Span that the derivation is built over - @JsonProperty public final String cat; - @JsonProperty public final int start; - @JsonProperty public final int end; + public final String cat; + public final int start; + public final int end; + public String canonicalUtterance; // If this derivation is composed of other derivations - @JsonProperty public final Rule rule; // Which rule was used to produce this derivation? Set to nullRule if not. - @JsonProperty public final List children; // Corresponds to the RHS of the rule. //// SemanticFn fields: read/written by SemanticFn. @@ -44,63 +45,46 @@ public class Derivation implements SemanticFn.Callable { // information. This could be its own class, but expose more right now to // be more flexible. - @JsonProperty public final Formula formula; // Logical form produced by this derivation - @JsonProperty public final SemType type; // Type corresponding to that logical form //// Fields produced by feature extractor, evaluation, etc. - final List localChoices = new ArrayList(); // Just for printing/debugging. + private List localChoices; // Just for printing/debugging. + + // TODO(pliang): make fields private // Information for scoring - @JsonProperty final private FeatureVector localFeatureVector; // Features - @JsonProperty double score = Double.NaN; // Weighted combination of features + private final FeatureVector localFeatureVector; // Features + double score = Double.NaN; // Weighted combination of features // What the formula evaluates to (optionally set later; only non-null for the root Derivation) - @JsonProperty Value value; - @JsonProperty Evaluation executorStats; + public Value value; + public Evaluation executorStats; // Number in [0, 1] denoting how correct the value is. - @JsonProperty double compatibility = Double.NaN; + public double compatibility = Double.NaN; // Probability (normalized exp of score). - @JsonProperty double prob = Double.NaN; + public double prob = Double.NaN; - // Miscellaneous statistics which are filled in by BeamParser - // TODO: wrap this in an object BeamParserStats + // Miscellaneous statistics int maxBeamPosition = -1; // Lowest position that this tree or any of its children is on the beam (after sorting) int maxUnsortedBeamPosition = -1; // Lowest position that this tree or any of its children is on the beam (before sorting) int preSortBeamPosition = -1; int postSortBeamPosition = -1; - //caching the hashcode + + // Cache the hash code int hashCode = -1; - // Initially, derivation is created with these parameters. - // Then, SemanticFn is called to create many copies of this derivation with different + // Each derivation that gets created gets a unique ID in increasing order so that + // we can break ties consistently for reproducible results. + long creationIndex; + public static long numCreated = 0; // Incremented for each derivation we create. + public static final Comparator derivScoreComparator = new ScoredDerivationComparator(); - // TODO: why do we need these constructors if we have the Builder? - /** Constructor for humans. */ - public Derivation(String cat, int start, int end, - Rule rule, - List children, - Formula formula, - SemType type) { - this.cat = cat; - this.start = start; - this.end = end; - this.rule = rule; - this.children = children; - this.formula = formula; - this.type = type; - this.localFeatureVector = new FeatureVector(); - } + public static final List emptyList = Collections.emptyList(); - /** Constructor for humans. */ - public Derivation(String cat, int start, int end, - Rule rule, - List children) { - this(cat, start, end, rule, children, null, null); - } + // A Derivation is built from /** Builder for everyone. */ public static class Builder { @@ -117,61 +101,25 @@ public class Derivation implements SemanticFn.Callable { private Evaluation executorStats; private double compatibility = Double.NaN; private double prob = Double.NaN; + private String canonicalUtterance = ""; + + public Builder cat(String cat) { this.cat = cat; return this; } + public Builder start(int start) { this.start = start; return this; } + public Builder end(int end) { this.end = end; return this; } + public Builder rule(Rule rule) { this.rule = rule; return this; } + public Builder children(List children) { this.children = children; return this; } + public Builder formula(Formula formula) { this.formula = formula; return this; } + public Builder type(SemType type) { this.type = type; return this; } + public Builder localFeatureVector(FeatureVector localFeatureVector) { this.localFeatureVector = localFeatureVector; return this; } + public Builder score(double score) { this.score = score; return this; } + public Builder value(Value value) { this.value = value; return this; } + public Builder executorStats(Evaluation executorStats) { this.executorStats = executorStats; return this; } + public Builder compatibility(double compatibility) { this.compatibility = compatibility; return this; } + public Builder prob(double prob) { this.prob = prob; return this; } + public Builder canonicalUtterance(String canonicalUtterance) { this.canonicalUtterance = canonicalUtterance; return this; } - public Builder cat(String cat) { - this.cat = cat; - return this; - } - public Builder start(int start) { - this.start = start; - return this; - } - public Builder end(int end) { - this.end = end; - return this; - } - public Builder rule(Rule rule) { - this.rule = rule; - return this; - } - public Builder children(List children) { - this.children = children; - return this; - } - public Builder formula(Formula formula) { - this.formula = formula; - return this; - } - public Builder type(SemType type) { - this.type = type; - return this; - } - public Builder localFeatureVector(FeatureVector localFeatureVector) { - this.localFeatureVector = localFeatureVector; - return this; - } - public Builder score(double score) { - this.score = score; - return this; - } - public Builder value(Value value) { - this.value = value; - return this; - } - public Builder executorStats(Evaluation executorStats) { - this.executorStats = executorStats; - return this; - } - public Builder compatibility(double compatibility) { - this.compatibility = compatibility; - return this; - } - public Builder prob(double prob) { - this.prob = prob; - return this; - } public Builder withStringFormulaFrom(String value) { - this.formula = new ValueFormula(new StringValue(value)); + this.formula = new ValueFormula<>(new StringValue(value)); this.type = SemType.stringType; return this; } @@ -180,6 +128,7 @@ public class Derivation implements SemanticFn.Callable { this.type = deriv.type; return this; } + public Builder withCallable(SemanticFn.Callable c) { this.cat = c.getCat(); this.start = c.getStart(); @@ -188,27 +137,16 @@ public class Derivation implements SemanticFn.Callable { this.children = c.getChildren(); return this; } + public Derivation createDerivation() { return new Derivation( cat, start, end, rule, children, formula, type, - localFeatureVector, score, value, executorStats, compatibility, prob); + localFeatureVector, score, value, executorStats, compatibility, prob, canonicalUtterance); } } - @JsonCreator - Derivation(@JsonProperty("cat") String cat, - @JsonProperty("start") int start, - @JsonProperty("end") int end, - @JsonProperty("rule") Rule rule, - @JsonProperty("children") List children, - @JsonProperty("formula") Formula formula, - @JsonProperty("type") SemType type, - @JsonProperty("localFeatureVector") FeatureVector localFeatureVector, - @JsonProperty("score") double score, - @JsonProperty("value") Value value, - @JsonProperty("executorStats") Evaluation executorStats, - @JsonProperty("compatibility") double compatibility, - @JsonProperty("prob") double prob) { + Derivation(String cat, int start, int end, Rule rule, List children, Formula formula, SemType type, + FeatureVector localFeatureVector, double score, Value value, Evaluation executorStats, double compatibility, double prob, String canonicalUtterance) { this.cat = cat; this.start = start; this.end = end; @@ -222,6 +160,8 @@ public class Derivation implements SemanticFn.Callable { this.executorStats = executorStats; this.compatibility = compatibility; this.prob = prob; + this.canonicalUtterance = canonicalUtterance; + this.creationIndex = numCreated++; } public Formula getFormula() { return formula; } @@ -230,11 +170,14 @@ public class Derivation implements SemanticFn.Callable { public double getCompatibility() { return compatibility; } public List getChildren() { return children; } public Value getValue() { return value; } + + public boolean isFeaturizedAndScored() { return !Double.isNaN(score); } public boolean isExecuted() { return value != null; } public int getMaxBeamPosition() { return maxBeamPosition; } public String getCat() { return cat; } public int getStart() { return start; } public int getEnd() { return end; } + public boolean containsIndex(int i) { return i < end && i >= start; } public Rule getRule() { return rule; } public Evaluation getExecutorStats() { return executorStats; } @@ -243,24 +186,31 @@ public class Derivation implements SemanticFn.Callable { return Formulas.getString(children.get(i).formula); } - // Return whether |deriv| is the top Derivation. + // Return whether |deriv| is built over the root Derivation. public boolean isRoot(int numTokens) { - return cat.equals(Rule.rootCat) && start == 0 && end == numTokens; + return cat.equals(Rule.rootCat) && ((start == 0 && end == numTokens) || (start == -1)); } // Functions that operate on features. public void addFeature(String domain, String name) { addFeature(domain, name, 1); } public void addFeature(String domain, String name, double value) { this.localFeatureVector.add(domain, name, value); } + public void addHistogramFeature(String domain, String name, double value, + int initBinSize, int numBins, boolean exp) { + this.localFeatureVector.addHistogram(domain, name, value, initBinSize, numBins, exp); + } public void addFeatureWithBias(String domain, String name, double value) { this.localFeatureVector.addWithBias(domain, name, value); } public void addFeatures(FeatureVector fv) { this.localFeatureVector.add(fv); } - public void addFeatures(Derivation deriv) { this.localFeatureVector.add(deriv.localFeatureVector); } + + public double localScore(Params params) { + return localFeatureVector.dotProduct(params); + } /** * Recursively compute the score for each node in derivation. Update |score| * field as well as return its value. */ public double computeScore(Params params) { - score = localFeatureVector.dotProduct(params); + score = localScore(params); if (children != null) for (Derivation child : children) score += child.computeScore(params); @@ -272,94 +222,63 @@ public class Derivation implements SemanticFn.Callable { * already scored). */ public double computeScoreLocal(Params params) { - score = localFeatureVector.dotProduct(params); + score = localScore(params); if (children != null) for (Derivation child : children) score += child.score; return score; } - public void ensureExecuted(Executor executor) { - if (!isExecuted()) { - StopWatchSet.begin("Executor.execute"); - Executor.Response response = executor.execute(formula); - StopWatchSet.end(); - value = response.value; - executorStats = response.stats; - } - } - - /** Copy execution result and stats from another Derivation. */ - public void setExecResults(Derivation deriv) { - if (isExecuted()) - throw new IllegalStateException("setExecResults() on an executed Derivation"); - value = deriv.value; - executorStats = deriv.executorStats; - // TODO: why do we need this? - localFeatureVector.add(deriv.localFeatureVector, DenotationFeatureMatcher.matcher); - } - - @Override - public boolean equals(Object thatObj) { - //if(thatObj == null || getClass() != thatObj.getClass()) return false; - Derivation that = (Derivation) thatObj; - if (!this.cat.equals(that.cat)) return false; - if (this.start != that.start) return false; - if (this.end != that.end) return false; - if (!this.rule.equals(that.rule)) return false; - if (!this.children.equals(that.children)) return false; - if (!this.formula.equals(that.formula)) return false; - return true; - } - - @Override - public int hashCode() { - if(hashCode==-1) { - int hash = 0x7ed55d16; - hash = hash * 0xd3a2646c + cat.hashCode(); - hash = hash * 0xd3a2646c + start; - hash = hash * 0xd3a2646c + end; - hash = hash * 0xd3a2646c + rule.hashCode(); - hash = hash * 0xd3a2646c + children.hashCode(); - hash = hash * 0xd3a2646c + formula.hashCode(); -// Boolean b = isCompleteDerivation(); -// hash = hash * 0xd3a2646c + b.hashCode(); - hashCode = hash; - } - return hashCode; + // If we haven't executed the formula associated with this derivation, then + // execute it! + public void ensureExecuted(Executor executor, ContextValue context) { + if (isExecuted()) return; + StopWatchSet.begin("Executor.execute"); + Executor.Response response = executor.execute(formula, context); + StopWatchSet.end(); + value = response.value; + executorStats = response.stats; } public LispTree toLispTree() { LispTree tree = LispTree.proto.newList(); tree.addChild("derivation"); - //tree.addChild(LispTree.proto.newList("span", cat+"["+start+":"+end+"]")); if (formula != null) tree.addChild(LispTree.proto.newList("formula", formula.toLispTree())); if (value != null) { if (opts.showValues) tree.addChild(LispTree.proto.newList("value", value.toLispTree())); - else if (value instanceof ListValue) - tree.addChild(((ListValue) value).values.size() + " values"); + else if (value instanceof ListValue) { + List values = ((ListValue) value).values; + if (opts.showFirstValue && values.size() > 0) { + tree.addChild(LispTree.proto.newList(values.size() + " values", values.get(0).toLispTree())); + } else { + tree.addChild(values.size() + " values"); + } + } + } - if (type != null) + if (type != null && opts.showTypes) tree.addChild(LispTree.proto.newList("type", type.toLispTree())); if (opts.showRules) { if (rule != null) tree.addChild(getRuleLispTree()); } + if (opts.showUtterance && canonicalUtterance != null) { + tree.addChild(LispTree.proto.newList("canonicalUtterance", canonicalUtterance)); + } return tree; } /** - * lisp tree showing the entire parse tree - * @return + * @return lisp tree showing the entire parse tree */ public LispTree toRecursiveLispTree() { LispTree tree = LispTree.proto.newList(); tree.addChild("derivation"); - tree.addChild(LispTree.proto.newList("span", cat+"["+start+":"+end+"]")); + tree.addChild(LispTree.proto.newList("span", cat + "[" + start + ":" + end + "]")); if (formula != null) tree.addChild(LispTree.proto.newList("formula", formula.toLispTree())); - for(Derivation child: children) + for (Derivation child : children) tree.addChild(child.toRecursiveLispTree()); return tree; } @@ -368,13 +287,13 @@ public class Derivation implements SemanticFn.Callable { return toRecursiveLispTree().toString(); } + // TODO(pliang): remove this in favor of localChoices private LispTree getRuleLispTree() { LispTree tree = LispTree.proto.newList(); tree.addChild("rules"); getRuleLispTreeRecurs(tree); return tree; } - private void getRuleLispTreeRecurs(LispTree tree) { if (children.size() > 0) { tree.addChild(LispTree.proto.newList("rule", rule.toLispTree())); @@ -389,6 +308,9 @@ public class Derivation implements SemanticFn.Callable { } public String toString() { return toLispTree().toString(); } + public void incrementLocalFeatureVector(double factor, Map map) { + localFeatureVector.increment(factor, map, AllFeatureMatcher.matcher); + } public void incrementAllFeatureVector(double factor, Map map) { incrementAllFeatureVector(factor, map, AllFeatureMatcher.matcher); } @@ -398,35 +320,41 @@ public class Derivation implements SemanticFn.Callable { child.incrementAllFeatureVector(factor, map, updateFeatureMatcher); } + // recursively renames all features in derivation by adding a prefix + public FeatureVector addPrefixLocalFeatureVector(String prefix) { + return localFeatureVector.addPrefix(prefix); + } + public Map getAllFeatureVector() { - Map m = new HashMap(); + Map m = new HashMap<>(); incrementAllFeatureVector(1.0d, m, AllFeatureMatcher.matcher); return m; } - // TODO: this is crazy inefficient - public Double getAllFeatureVector(String featureName) { - Map m = new HashMap(); + // TODO(pliang): this is crazy inefficient + public double getAllFeatureVector(String featureName) { + Map m = new HashMap<>(); incrementAllFeatureVector(1.0d, m, new ExactFeatureMatcher(featureName)); - return MapUtils.get(m,featureName,0.0); + return MapUtils.get(m, featureName, 0.0); + } + + public void addLocalChoice(String choice) { + if (localChoices == null) + localChoices = new ArrayList(); + localChoices.add(choice); } public void incrementAllChoices(int factor, Map map) { if (opts.showRules) MapUtils.incr(map, "[" + start + ":" + end + "] " + rule.toString(), 1); - for (String choice : localChoices) - MapUtils.incr(map, choice, factor); + if (localChoices != null) { + for (String choice : localChoices) + MapUtils.incr(map, choice, factor); + } for (Derivation child : children) child.incrementAllChoices(factor, map); } - //methods added to allow checking if a derivation is complete and completing it - public boolean isCompleteDerivation() {return true;} - public List complete(Example ex) { - throw new RuntimeException("Can not complete a derivation that is already complete, use isCompleteDerivation()"); - } - - // Used to compare derivations by score. public static class ScoredDerivationComparator implements Comparator { @Override @@ -434,14 +362,37 @@ public class Derivation implements SemanticFn.Callable { if (deriv1.score > deriv2.score) return -1; if (deriv1.score < deriv2.score) return +1; // Ensure reproducible randomness - if (deriv1.hashCode() < deriv2.hashCode()) return -1; - if (deriv1.hashCode() > deriv2.hashCode()) return +1; + if (deriv1.creationIndex < deriv2.creationIndex) return -1; + if (deriv1.creationIndex > deriv2.creationIndex) return +1; return 0; } } + // Used to compare derivations by compatibility. + public static class CompatibilityDerivationComparator implements Comparator { + @Override + public int compare(Derivation deriv1, Derivation deriv2) { + if (deriv1.compatibility > deriv2.compatibility) return -1; + if (deriv1.compatibility < deriv2.compatibility) return +1; + // Ensure reproducible randomness + if (deriv1.creationIndex < deriv2.creationIndex) return -1; + if (deriv1.creationIndex > deriv2.creationIndex) return +1; + return 0; + } + } + + // for debugging + public void printDerivationRecursively() { + LogInfo.logs("Deriv: %s(%s,%s) %s", cat, start, end, formula); + for (int i = 0; i < children.size(); i++) { + LogInfo.begin_track("child %s:", i); + children.get(i).printDerivationRecursively(); + LogInfo.end_track(); + } + } + public static void sortByScore(List trees) { - Collections.sort(trees, new ScoredDerivationComparator()); + Collections.sort(trees, derivScoreComparator); } // Generate a probability distribution over derivations given their scores. diff --git a/src/edu/stanford/nlp/sempre/DerivationConstraint.java b/src/edu/stanford/nlp/sempre/DerivationConstraint.java deleted file mode 100644 index bd897ed..0000000 --- a/src/edu/stanford/nlp/sempre/DerivationConstraint.java +++ /dev/null @@ -1,28 +0,0 @@ -package edu.stanford.nlp.sempre; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; - -/** - * A DerivationConstraint filters the set of Derivations. Currently, it's just - * used to visualize the Derivations. - */ -public class DerivationConstraint { - // Regular expression on formula's toString() - private final String formulaPattern; - - @JsonCreator - public DerivationConstraint(String formulaPattern) { - this.formulaPattern = formulaPattern; - } - - // Intended just for JsonValue. - @JsonValue - public String getFormulaPattern() { return formulaPattern; } - - // Return the factor - public boolean satisfies(Example ex, Derivation deriv) { - //LogInfo.logs("satisfies: %s %s", deriv, formulaPattern); - return deriv.getFormula().toString().matches(".*" + formulaPattern + ".*"); - } -} diff --git a/src/edu/stanford/nlp/sempre/DerivationStream.java b/src/edu/stanford/nlp/sempre/DerivationStream.java new file mode 100644 index 0000000..3443860 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/DerivationStream.java @@ -0,0 +1,13 @@ +package edu.stanford.nlp.sempre; + +import java.util.Iterator; + +/** + * Represents a stream of Derivations which are constructed lazily for efficiency. + * Use either SingleDerivationStream or MultipleDerivationStream. + * Created by joberant on 3/14/14. + */ +public interface DerivationStream extends Iterator { + Derivation peek(); + int estimatedSize(); +} diff --git a/src/edu/stanford/nlp/sempre/DescriptionValue.java b/src/edu/stanford/nlp/sempre/DescriptionValue.java index 8811d06..d0517cc 100644 --- a/src/edu/stanford/nlp/sempre/DescriptionValue.java +++ b/src/edu/stanford/nlp/sempre/DescriptionValue.java @@ -1,7 +1,6 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; -import fig.basic.Option; /** * Represents the description part of a NameValue ("Barack Obama" rather than @@ -10,11 +9,6 @@ import fig.basic.Option; * @author Andrew Chou */ public class DescriptionValue extends Value { - public static class Options { - @Option(gloss = "Verbose.") public boolean verbose = false; - } - public static Options opts = new Options(); - public final String value; public DescriptionValue(LispTree tree) { this(tree.child(1).value); } @@ -27,18 +21,11 @@ public class DescriptionValue extends Value { return tree; } - public double getCompatibility(Value thatValue) { - // Match the description part of NameValue. - if (thatValue instanceof NameValue) - return value.equals(((NameValue)thatValue).description) ? 1 : 0; - - return super.getCompatibility(thatValue); - } - @Override public int hashCode() { return value.hashCode(); } - @Override public boolean equals(Object thatObj) { - if (!(thatObj instanceof DescriptionValue)) return false; - DescriptionValue that = (DescriptionValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DescriptionValue that = (DescriptionValue) o; return this.value.equals(that.value); } } diff --git a/src/edu/stanford/nlp/sempre/ErrorValue.java b/src/edu/stanford/nlp/sempre/ErrorValue.java index 6825023..51da86f 100644 --- a/src/edu/stanford/nlp/sempre/ErrorValue.java +++ b/src/edu/stanford/nlp/sempre/ErrorValue.java @@ -31,6 +31,9 @@ public class ErrorValue extends Value { // Server returned something back but it had a bad format (e.g., HTML instead of XML). public static final ErrorValue badFormat = new ErrorValue("BADFORMAT"); + // Execution of Java failed (generated by JavaExecutor). + public static final ErrorValue badJava(String message) { return new ErrorValue("BADJAVA: " + message); } + public final String type; public ErrorValue(LispTree tree) { this.type = tree.child(1).value; } @@ -44,6 +47,7 @@ public class ErrorValue extends Value { } @Override + // TODO(pliang): return this (error type) to avoid clashes with NameValue public String toString() { return type; } public static ErrorValue fromString(String s) { if (s.equals(timeout.type)) return timeout; @@ -54,10 +58,6 @@ public class ErrorValue extends Value { return null; } - public double getCompatibility(Value thatValue) { - return 0; // Never give points for error. - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/edu/stanford/nlp/sempre/Evaluation.java b/src/edu/stanford/nlp/sempre/Evaluation.java deleted file mode 100644 index 1fc2e07..0000000 --- a/src/edu/stanford/nlp/sempre/Evaluation.java +++ /dev/null @@ -1,130 +0,0 @@ -package edu.stanford.nlp.sempre; - -import com.fasterxml.jackson.annotation.JsonValue; -import com.google.common.collect.Maps; -import fig.basic.Fmt; -import fig.basic.LispTree; -import fig.basic.LogInfo; -import fig.basic.StatFig; -import fig.exec.Execution; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * An Evaluation measures how well the system is doing on a set of examples. - * Formally, it is just a collection of arbitrary statistics. - * - * @author Percy Liang - */ -public class Evaluation { - private List names = new ArrayList(); - private List values = new ArrayList(); - - public StatFig getFig(String name) { - int i = names.indexOf(name); - if (i == -1) - return null; - return values.get(i); - } - - // Methods to add new metrics to evaluation and aggregate evaluations. - private StatFig getFigHard(String name) { - int i = names.indexOf(name); - if (i == -1) { - i = names.size(); - names.add(name); - values.add(new StatFig()); - } - return values.get(i); - } - - public void add(String name, boolean value) { add(name, value ? 1 : 0); } - public void add(String name, double value) { add(name, null, value); } - public void add(String name, Object key, double value) { - StatFig fig = new StatFig(); - fig.add(key, value); - add(name, fig); - } - public void add(String name, StatFig fig) { - getFigHard(name).add(fig); - } - public synchronized void add(Evaluation eval) { - for (int i = 0; i < eval.names.size(); i++) - add(eval.names.get(i), eval.values.get(i)); - } - - @Deprecated - public LispTree toLispTree() { - LispTree out = LispTree.proto.newList(); - for (int i = 0; i < names.size(); i++) { - out.addChild(LispTree.proto.newList(names.get(i), Fmt.D(values.get(i).mean()))); - } - return out; - } - - @Deprecated - public static Evaluation fromLispTree(LispTree t) { - Evaluation e = new Evaluation(); - for (int i = 0; i < t.children.size(); i++) { - e.add( - t.child(i).child(0).value, - Double.parseDouble(t.child(i).child(1).value)); - } - return e; - } - - public String summary() { return summary(" "); } - public String summary(String delim) { - StringBuilder out = new StringBuilder(); - for (int i = 0; i < names.size(); i++) { - if (i > 0) out.append(delim); - out.append(names.get(i) + '=' + Fmt.D(values.get(i).mean())); - } - return out.toString(); - } - - private void putOutput(String prefix, StatFig fig) { - Execution.putOutput(prefix + ".count", fig.count()); - if (fig.count() > 0) { - Execution.putOutput(prefix + ".mean", fig.mean()); - Execution.putOutput(prefix + ".max", fig.max()); - } - } - - private String basePrefix(String prefix) { - String[] parts = prefix.split("\\."); return parts[parts.length - 1]; - } - - public void putOutput(String prefix) { - for (int i = 0; i < names.size(); i++) - putOutput(basePrefix(prefix) + "." + names.get(i), values.get(i)); - } - public void logStats(String prefix) { - LogInfo.begin_track_printAll("Evaluation stats for %s", prefix); - for (int i = 0; i < names.size(); i++) - LogInfo.log(names.get(i) + " = " + values.get(i)); - LogInfo.end_track(); - } - - @JsonValue - Map> toMap() { - Map> m = Maps.newHashMapWithExpectedSize(names.size()); - for (int i = 0; i < names.size(); i++) - m.put(names.get(i), statFigToMap(values.get(i))); - return m; - } - - private static Map statFigToMap(StatFig fig) { - Map m = Maps.newHashMapWithExpectedSize(7); - m.put("min", fig.min()); - m.put("minKey", fig.minKey()); - m.put("max", fig.max()); - m.put("maxKey", fig.maxKey()); - m.put("mean", fig.mean()); - m.put("stddev", fig.stddev()); - m.put("count", fig.count()); - return m; - } -} diff --git a/src/edu/stanford/nlp/sempre/ExactValueEvaluator.java b/src/edu/stanford/nlp/sempre/ExactValueEvaluator.java new file mode 100644 index 0000000..75f67e9 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ExactValueEvaluator.java @@ -0,0 +1,8 @@ +package edu.stanford.nlp.sempre; + +// This is the simplest evaluator, but exact match can sometimes be too harsh. +public class ExactValueEvaluator implements ValueEvaluator { + public double getCompatibility(Value target, Value pred) { + return target.equals(pred) ? 1 : 0; + } +} diff --git a/src/edu/stanford/nlp/sempre/Example.java b/src/edu/stanford/nlp/sempre/Example.java index d28ca36..496545b 100644 --- a/src/edu/stanford/nlp/sempre/Example.java +++ b/src/edu/stanford/nlp/sempre/Example.java @@ -1,8 +1,12 @@ package edu.stanford.nlp.sempre; -import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.collect.Sets; +import fig.basic.Evaluation; import fig.basic.LispTree; import fig.basic.LogInfo; @@ -18,14 +22,9 @@ import java.util.List; * @author Percy Liang * @author Roy Frostig */ -@JsonIgnoreProperties(ignoreUnknown=true) +@JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class Example { - public static class JsonViews { - public static class WithDerivations {} - public static class WithDPChart {} - } - //// Information from the input file. // Unique identifier for this example. @@ -34,181 +33,96 @@ public class Example { // Input utterance @JsonProperty public final String utterance; - // Provides - @JsonProperty public final DerivationConstraint derivConstraint; + // Context + @JsonProperty public ContextValue context; // What we should try to predict. - @JsonProperty public Formula targetFormula; // Logical form - @JsonProperty public Value targetValue; // Answer + @JsonProperty public Formula targetFormula; // Logical form (e.g., database query) + @JsonProperty public Value targetValue; // Denotation (e.g., answer) //// Information after preprocessing (e.g., tokenization, POS tagging, NER, syntactic parsing, etc.). @JsonProperty public LanguageInfo languageInfo = null; - // Tokens come from languageInfo, but if we don't have one, - // they go here (usually due to deserialization). - // DELETE - private List backupTokens = null; - //// Output of the parser. - // Predicted derivations: sorted by score. - @JsonProperty @JsonView(JsonViews.WithDerivations.class) - List predDerivations; + // Predicted derivations (sorted by score). + public List predDerivations; - // To debug amount of ordering change due to executing and obtaining - // denotation features. - List predDerivationsAfterParse; - - // Statistics about how well we did during parsing and execution. - private Evaluation parseEvaluation; - private Evaluation evaluation; - - // Maximum position in any cell of the chart of any sub-derivation of a - // correct derivation. Under the current parameters, we would need to set - // the beam size to at least this to get it right. - int correctMaxBeamPosition = -1; - // Beam size to use for parsing - int beamSize = -1; + // Statistics relating to processing the example. + public Evaluation evaluation; public static class Builder { private String id; private String utterance; - private DerivationConstraint derivConstraint; + private ContextValue context; private Formula targetFormula; private Value targetValue; private List predDerivations; private LanguageInfo languageInfo; - public Builder setId(String id) { - this.id = id; - return this; - } - public Builder setUtterance(String utterance) { - this.utterance = utterance; - return this; - } - public Builder setDerivConstraint(DerivationConstraint derivConstraint) { - this.derivConstraint = derivConstraint; - return this; - } - public Builder setTargetFormula(Formula targetFormula) { - this.targetFormula = targetFormula; - return this; - } - public Builder setTargetValue(Value targetValue) { - this.targetValue = targetValue; - return this; - } - public Builder setPredDerivations(List predDerivations) { - this.predDerivations = predDerivations; - return this; - } - public Builder setLanguageInfo(LanguageInfo languageInfo) { - this.languageInfo = languageInfo; - return this; - } + public Builder setId(String id) { this.id = id; return this; } + public Builder setUtterance(String utterance) { this.utterance = utterance; return this; } + public Builder setContext(ContextValue context) { this.context = context; return this; } + public Builder setTargetFormula(Formula targetFormula) { this.targetFormula = targetFormula; return this; } + public Builder setTargetValue(Value targetValue) { this.targetValue = targetValue; return this; } + public Builder setLanguageInfo(LanguageInfo languageInfo) { this.languageInfo = languageInfo; return this; } public Builder withExample(Example ex) { setId(ex.id); setUtterance(ex.utterance); - setDerivConstraint(ex.derivConstraint); + setContext(ex.context); setTargetFormula(ex.targetFormula); setTargetValue(ex.targetValue); - setPredDerivations(ex.predDerivations); return this; } - public Example createExample() { - return new Example( - id, utterance, derivConstraint, targetFormula, - targetValue, predDerivations, languageInfo); + return new Example(id, utterance, context, targetFormula, targetValue, languageInfo); } } @JsonCreator public Example(@JsonProperty("id") String id, @JsonProperty("utterance") String utterance, - @JsonProperty("derivConstraint") DerivationConstraint derivConstraint, + @JsonProperty("context") ContextValue context, @JsonProperty("targetFormula") Formula targetFormula, @JsonProperty("targetValue") Value targetValue, - @JsonProperty("predDerivations") List predDerivations, @JsonProperty("languageInfo") LanguageInfo languageInfo) { this.id = id; this.utterance = utterance; - this.derivConstraint = derivConstraint; + this.context = context; this.targetFormula = targetFormula; this.targetValue = targetValue; - this.predDerivations = predDerivations; this.languageInfo = languageInfo; } // Accessors public String getId() { return id; } public String getUtterance() { return utterance; } - public Evaluation getEvaluation() { return evaluation; } public int numTokens() { return languageInfo.tokens.size(); } public List getPredDerivations() { return predDerivations; } - public void setTargetFormula(Formula targetFormula) { - this.targetFormula = targetFormula; - } - public void setTargetValue(Value targetValue) { - this.targetValue = targetValue; - } + + public void setContext(ContextValue context) { this.context = context; } + public void setTargetFormula(Formula targetFormula) { this.targetFormula = targetFormula; } + public void setTargetValue(Value targetValue) { this.targetValue = targetValue; } public String spanString(int start, int end) { - return String.format("%d:%d[%s]", start, end, phraseString(start, end)); + return String.format("%d:%d[%s]", start, end, start != -1 ? phraseString(start, end) : "..."); } public String phraseString(int start, int end) { return Joiner.on(' ').join(languageInfo.tokens.subList(start, end)); } // Return a string representing the tokens between start and end. - public List getTokens() { - return (languageInfo != null) ? languageInfo.tokens : backupTokens; - } + public List getTokens() { return languageInfo.tokens; } public List getLemmaTokens() { return languageInfo.lemmaTokens; } public String token(int i) { return languageInfo.tokens.get(i); } public String lemmaToken(int i) { return languageInfo.lemmaTokens.get(i); } public String posTag(int i) { return languageInfo.posTags.get(i); } - public String phrase(int start, int end) { - return languageInfo.phrase(start, end); - } - public String lemmaPhrase(int start, int end) { - return languageInfo.lemmaPhrase(start, end); - } + public String phrase(int start, int end) { return languageInfo.phrase(start, end); } + public String lemmaPhrase(int start, int end) { return languageInfo.lemmaPhrase(start, end); } - void setParseEvaluation(Evaluation eval) { parseEvaluation = eval; } - public void setEvaluation(Evaluation eval) { evaluation = eval; } + public String toJson() { return Json.writeValueAsStringHard(this); } + public static Example fromJson(String json) { return Json.readValueHard(json, Example.class); } - public Evaluation computeTotalEvaluation() { - Evaluation eval = new Evaluation(); - if (parseEvaluation != null) - eval.add(parseEvaluation); - if (evaluation != null) - eval.add(evaluation); - return eval; - } - - void rescoreAndSortPredDerivations(Params params) { - for (Derivation deriv : predDerivations) - deriv.computeScore(params); - Derivation.sortByScore(predDerivations); - } - - public String toJson() { - return Json.writeValueAsStringHard(this); - } - - public static Example fromJson(String json) { - return Json.readValueHard(json, Example.class); - } - - /** Use JSON instead. */ - @Deprecated - public static Example fromLispTree(LispTree tree) { - return fromLispTree(tree, null); - } - - @Deprecated public static Example fromLispTree(LispTree tree, String defaultId) { Builder b = new Builder().setId(defaultId); @@ -225,8 +139,11 @@ public class Example { if (arg.children.size() != 2) throw new RuntimeException("Expect one target value"); b.setTargetValue(Values.fromLispTree(arg.child(1))); + } else if ("context".equals(label)) { + b.setContext(new ContextValue(arg)); } } + b.setLanguageInfo(new LanguageInfo()); Example ex = b.createExample(); @@ -237,18 +154,15 @@ public class Example { // Do nothing } else if ("tokens".equals(label)) { int n = arg.child(1).children.size(); - ex.backupTokens = new ArrayList(n); for (int j = 0; j < n; j++) - ex.backupTokens.add(arg.child(1).child(j).value); + ex.languageInfo.tokens.add(arg.child(1).child(j).value); } else if ("evaluation".equals(label)) { ex.evaluation = Evaluation.fromLispTree(arg.child(1)); - } else if ("parseEvaluation".equals(label)) { - ex.parseEvaluation = Evaluation.fromLispTree(arg.child(1)); } else if ("predDerivations".equals(label)) { - ex.predDerivations = new ArrayList(); + ex.predDerivations = new ArrayList<>(); for (int j = 1; j < arg.children.size(); j++) ex.predDerivations.add(derivationFromLispTree(arg.child(j))); - } else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues").contains(label)) { + } else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues", "context").contains(label)) { throw new RuntimeException("Invalid example argument: " + arg); } } @@ -256,11 +170,8 @@ public class Example { return ex; } - public void preprocess() { - if (this.languageInfo == null) - this.languageInfo = new LanguageInfo(); - this.languageInfo.analyze(this.utterance); - this.log(); + public void preprocess(LanguageAnalyzer analyzer) { + this.languageInfo = analyzer.analyze(this.utterance); } public void log() { @@ -270,20 +181,32 @@ public class Example { LogInfo.logs("POS tags: %s", languageInfo.posTags); LogInfo.logs("NER tags: %s", languageInfo.nerTags); LogInfo.logs("NER values: %s", languageInfo.nerValues); + if (context != null) + LogInfo.logs("context: %s", context); if (targetFormula != null) LogInfo.logs("targetFormula: %s", targetFormula); if (targetValue != null) LogInfo.logs("targetValue: %s", targetValue); + LogInfo.logs("Dependency children: %s", languageInfo.dependencyChildren); LogInfo.end_track(); } + // To save memory public void clearPredDerivations() { predDerivations.clear(); - predDerivationsAfterParse.clear(); } - /** Use JSON serialization instead. */ - @Deprecated + public List getCorrectDerivations() { + List res = new ArrayList<>(); + for (Derivation deriv : predDerivations) { + if (deriv.compatibility == Double.NaN) + throw new RuntimeException("Compatibility is not set"); + if (deriv.compatibility > 0) + res.add(deriv); + } + return res; + } + public LispTree toLispTree(boolean outputPredDerivations) { LispTree tree = LispTree.proto.newList(); tree.addChild("example"); @@ -308,8 +231,6 @@ public class Example { if (evaluation != null) tree.addChild(LispTree.proto.newList("evaluation", evaluation.toLispTree())); - if (parseEvaluation != null) - tree.addChild(LispTree.proto.newList("parseEvaluation", parseEvaluation.toLispTree())); if (predDerivations != null && outputPredDerivations) { LispTree list = LispTree.proto.newList(); @@ -322,7 +243,6 @@ public class Example { return tree; } - @Deprecated private static Derivation derivationFromLispTree(LispTree item) { Derivation.Builder b = new Derivation.Builder() .cat(Rule.rootCat) @@ -352,12 +272,9 @@ public class Example { return b.createDerivation(); } - @Deprecated private static LispTree derivationToLispTree(Derivation deriv) { LispTree item = LispTree.proto.newList(); - // TODO: label scores and compatibilities in derivations to make output - // more self-documenting. item.addChild(deriv.compatibility + ""); item.addChild(deriv.prob + ""); item.addChild(deriv.score + ""); @@ -367,7 +284,7 @@ public class Example { item.addChild("null"); item.addChild(deriv.formula.toLispTree()); - HashMap features = new HashMap(); + HashMap features = new HashMap<>(); deriv.incrementAllFeatureVector(1, features); item.addChild(LispTree.proto.newList(features)); diff --git a/src/edu/stanford/nlp/sempre/ExampleUtils.java b/src/edu/stanford/nlp/sempre/ExampleUtils.java new file mode 100644 index 0000000..c4c79dd --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ExampleUtils.java @@ -0,0 +1,63 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.*; +import fig.exec.Execution; + +import java.io.*; +import java.util.*; + +/** + * Output examples in various forms. + * + * @author Percy Liang + */ +public final class ExampleUtils { + private ExampleUtils() { } + + // Output JSON file with just the basic input/output. + public static void writeJson(List examples) { + PrintWriter out = IOUtils.openOutHard(Execution.getFile("examples.json")); + for (Example ex : examples) + out.println(ex.toJson()); + out.close(); + } + + private static String escapeSpace(String s) { + return s.replaceAll(" ", " "); + } + + // Output examples in Simple Dataset Format (Ranking). + public static void writeSDF(int iter, String group, + Evaluation evaluation, + List examples, + boolean outputPredDerivations) { + String basePath = "preds-iter" + iter + "-" + group + ".examples"; + String outPath = Execution.getFile(basePath); + if (outPath == null || examples.size() == 0) return; + LogInfo.begin_track("Writing examples to %s", basePath); + PrintWriter out = IOUtils.openOutHard(outPath); + + LispTree p = LispTree.proto; + out.println("# " + p.L(p.L("iter", iter), p.L("group", group), p.L("numExamples", examples.size()), p.L("evaluation", evaluation.toLispTree()))); + for (Example ex : examples) { + out.println(""); + out.println("example " + ex.id); + out.println("description " + p.L(p.L("utterance", ex.utterance), p.L("targetValue", ex.targetValue.toLispTree()), p.L("evaluation", ex.evaluation.toLispTree()))); + + if (outputPredDerivations) { + for (Derivation deriv : ex.predDerivations) { + StringBuilder buf = new StringBuilder(); + buf.append("item " + escapeSpace(p.L(p.L("formula", deriv.formula.toLispTree()), p.L("value", deriv.value.toLispTree())).toString())); // Description + buf.append(" " + deriv.compatibility); + Map features = deriv.getAllFeatureVector(); + for (Map.Entry e : features.entrySet()) { + buf.append(" " + escapeSpace(e.getKey()) + ":" + e.getValue()); + } + out.println(buf.toString()); + } + } + } + out.close(); + LogInfo.end_track(); + } +} diff --git a/src/edu/stanford/nlp/sempre/Executor.java b/src/edu/stanford/nlp/sempre/Executor.java index c6bd1b6..5d8419a 100644 --- a/src/edu/stanford/nlp/sempre/Executor.java +++ b/src/edu/stanford/nlp/sempre/Executor.java @@ -1,5 +1,7 @@ package edu.stanford.nlp.sempre; +import fig.basic.Evaluation; + /** * An Executor takes a logical form (Formula) and computes its denotation * (Value). @@ -17,6 +19,6 @@ public abstract class Executor { public final Evaluation stats; } - // Execute the formula on the database. - public abstract Response execute(Formula formula); + // Execute the formula in the given context. + public abstract Response execute(Formula formula, ContextValue context); } diff --git a/src/edu/stanford/nlp/sempre/FeatureComputer.java b/src/edu/stanford/nlp/sempre/FeatureComputer.java new file mode 100644 index 0000000..4fdace4 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/FeatureComputer.java @@ -0,0 +1,25 @@ +package edu.stanford.nlp.sempre; + +/** + * A feature computer. + * + * Look at a derivation and add features to the feature vector. + * A FeatureComputer should be stateless. + * + * Before computing features, a FeatureComputer should call + * + * if (!FeatureExtractor.containsDomain(...)) return; + * + * to check the feature domain first. + */ +public interface FeatureComputer { + + /** + * This function is called on every sub-Derivation. + * + * It should extract only the features which depend in some way on |deriv|, + * not just on its children. + */ + void extractLocal(Example ex, Derivation deriv); + +} diff --git a/src/edu/stanford/nlp/sempre/FeatureExtractor.java b/src/edu/stanford/nlp/sempre/FeatureExtractor.java index a1af549..8040f9b 100644 --- a/src/edu/stanford/nlp/sempre/FeatureExtractor.java +++ b/src/edu/stanford/nlp/sempre/FeatureExtractor.java @@ -1,19 +1,23 @@ package edu.stanford.nlp.sempre; +import java.util.*; import com.google.common.base.Joiner; import com.google.common.collect.Sets; -import fig.basic.Option; -import fig.basic.StopWatchSet; - -import java.util.LinkedList; -import java.util.List; -import java.util.Set; +import fig.basic.*; /** * A FeatureExtractor specifies a mapping from derivations to feature vectors. - * There are two ways to add features to a Derivation: - * 1) Add FeatureExtractor classes, which are called on each sub-Derivation. - * 2) Use SemanticFn, which are called whenever they are called. + * + * [Using features] + * - Specify the feature domains in the featureDomains option. + * - If the features are defined in a separate FeatureComputer class, also specify + * the class name in the featureComputers option. + * + * [Implementing new features] There are 3 ways to implement new features: + * 1) Define features in SemanticFn, which is called when the SemanticFn is called. + * 2) Create a FeatureComputer class, which is called on each sub-Derivation if the class name + * is specified in the featureComputers option. + * 3) Add a method to this class. The method is called on each sub-Derivation. * * @author Percy Liang */ @@ -21,22 +25,25 @@ public class FeatureExtractor { // Global place to specify features. public static class Options { @Option(gloss = "Set of feature domains to include") - public Set featureDomains = Sets.newHashSet(); - + public Set featureDomains = new HashSet<>(); + @Option(gloss = "Set of feature computer classes to load") + public Set featureComputers = Sets.newHashSet("DerivOpCountFeatureComputer"); @Option(gloss = "Disable denotation features") public boolean disableDenotationFeatures = false; - @Option(gloss = "Use all possible features, regardless of what featureDomains says") public boolean useAllFeatures = false; - @Option(gloss = "Whether to conjoin all lemmas with binaries or each lemma") - public boolean conjoinAllLemmas = false; + @Option(gloss = "For bigram features in paraphrased utterances, maximum distance to consider") + public int maxBigramDistance = 3; } private Executor executor; - private DerivOpCountFeatureExtractor opCountExtractor = new DerivOpCountFeatureExtractor(); + private List featureComputers = new ArrayList<>(); public FeatureExtractor(Executor executor) { this.executor = executor; + for (String featureComputer : opts.featureComputers) { + featureComputers.add((FeatureComputer) Utils.newInstanceHard(SempreUtils.resolveClassName(featureComputer))); + } } public static Options opts = new Options(); @@ -51,14 +58,19 @@ public class FeatureExtractor { public void extractLocal(Example ex, Derivation deriv) { StopWatchSet.begin("FeatureExtractor.extractLocal"); extractRuleFeatures(ex, deriv); + extractSpanFeatures(ex, deriv); extractDenotationFeatures(ex, deriv); + extractDependencyFeatures(ex, deriv); extractWhTypeFeatures(ex, deriv); - opCountExtractor.extractLocal(ex, deriv); conjoinLemmaAndBinary(ex, deriv); + extractParaphraseFeatures(ex, deriv); + extractBigramFeatures(ex, deriv); + for (FeatureComputer featureComputer : featureComputers) + featureComputer.extractLocal(ex, deriv); StopWatchSet.end(); } - // Add an indicator for each rule that's applied. + // Add an indicator for each applied rule. void extractRuleFeatures(Example ex, Derivation deriv) { if (!containsDomain("rule")) return; if (deriv.rule != Rule.nullRule) { @@ -67,38 +79,96 @@ public class FeatureExtractor { } } + // Extract features on the linguistic information of the spanned (anchored) tokens. + // (Not applicable for floating rules) + void extractSpanFeatures(Example ex, Derivation deriv) { + if (!containsDomain("span") || deriv.start == -1) return; + deriv.addFeature("span", "cat=" + deriv.cat + ",#tokens=" + (deriv.end - deriv.start)); + deriv.addFeature("span", "cat=" + deriv.cat + ",POS=" + ex.posTag(deriv.start) + "..." + ex.posTag(deriv.end - 1)); + } + // Extract features on the denotation of the logical form produced. + // (For example, number of items in the list) void extractDenotationFeatures(Example ex, Derivation deriv) { if (!containsDomain("denotation")) return; if (!deriv.isRoot(ex.numTokens())) return; - deriv.ensureExecuted(executor); + deriv.ensureExecuted(executor, ex.context); if (deriv.value instanceof ErrorValue) { deriv.addFeature("denotation", "error"); return; } - if (!(deriv.value instanceof ListValue)) - throw new RuntimeException("Derivation value is not a list: " + deriv.value); + if (deriv.value instanceof ListValue) { + ListValue list = (ListValue) deriv.value; - ListValue list = (ListValue) deriv.value; - - if (Formulas.isCountFormula(deriv.formula)) { - // TODO: clean this up - if (list.values.size() != 1) { - throw new RuntimeException( - "Evaluation of count formula " + deriv.formula + " has size " + list.values.size()); + // TODO(pliang): this is hacky; don't depend on the logical form + if (Formulas.isCountFormula(deriv.formula)) { + if (list.values.size() != 1) { + deriv.addFeature("denotation", "size", list.values.size()); + } else { + int count = getNumber(list.values.get(0)); + deriv.addFeature("denotation", "count-size" + (count == 0 ? "=0" : ">0")); + } + } else { + int size = list.values.size(); + deriv.addFeature("denotation", "size" + (size < 3 ? "=" + size : ">=" + 3)); } - int count = (int)((NumberValue)list.values.get(0)).value; - deriv.addFeature("denotation", "count-size" + (count == 0 ? "=0" : ">0")); - } else { - int size = list.values.size(); - deriv.addFeature("denotation", "size" + (size < 3 ? "=" + size : ">=" + 3)); } } - // Conjunction of wh and type + int getNumber(Value value) { + if (value instanceof NumberValue) return (int) ((NumberValue) value).value; + if (value instanceof ListValue) return getNumber(((ListValue) value).values.get(0)); + throw new RuntimeException("Can't extract number from " + value); + } + + // Add an indicator for each alignment between a syntactic dependency (produced by the + // Stanford dependency parser) and the application of a semantic function. + void extractDependencyFeatures(Example ex, Derivation deriv) { + if (!containsDomain("dependencyParse") && !containsDomain("fullDependencyParse")) return; + if (deriv.rule != Rule.nullRule) { + for (Derivation child : deriv.children) { + for (int i = child.start; i < child.end; i++) { + for (LanguageInfo.DependencyEdge dependency : ex.languageInfo.dependencyChildren.get(i)) { + if (!child.containsIndex(dependency.modifier)) { + String direction = dependency.modifier > i ? "forward" : "backward"; + String containment = deriv.containsIndex(dependency.modifier) ? "internal" : "external"; + if (containsDomain("fullDependencyParse")) + addAllDependencyFeatures(dependency, direction, containment, + deriv); + else + deriv.addFeature("dependencyParse", + "(" + dependency.label + " " + direction + " " + containment + ") --- " + + deriv.getRule().toString()); + } + } + } + } + } + } + + private void addAllDependencyFeatures(LanguageInfo.DependencyEdge dependency, + String direction, String containment, Derivation deriv) { + String[] types = {dependency.label, "*"}; + String[] directions = {" " + direction, ""}; + String[] containments = {" " + containment, ""}; + String[] rules = {deriv.getRule().toString(), ""}; + for (String typePresent : types) { + for (String directionPresent : directions) { + for (String containmentPresent : containments) { + for (String rulePresent : rules) { + deriv.addFeature("fullDependencyParse", + "(" + typePresent + directionPresent + containmentPresent + ") --- " + rulePresent); + } + } + } + } + } + + // Conjunction of wh-question word and type + // (For example, "who" should go with PERSON and not DATE) void extractWhTypeFeatures(Example ex, Derivation deriv) { if (!containsDomain("whType")) return; if (!deriv.isRoot(ex.numTokens())) return; @@ -106,61 +176,172 @@ public class FeatureExtractor { if (ex.posTag(0).startsWith("W")) { deriv.addFeature("whType", "token0=" + ex.token(0) + "," + - "type=" + FreebaseInfo.getSingleton().coarseType(deriv.type.toString())); + "type=" + coarseType(deriv.type.toString())); } } + public static final String PERSON = "fb:people.person"; + public static final String LOC = "fb:location.location"; + public static final String ORG = "fb:organization.organization"; + + public static String coarseType(String type) { + Set superTypes = SemTypeHierarchy.singleton.getSupertypes(type); + if (superTypes != null) { + if (superTypes.contains(PERSON)) return PERSON; + if (superTypes.contains(LOC)) return LOC; + if (superTypes.contains(ORG)) return ORG; + if (superTypes.contains(CanonicalNames.NUMBER)) return CanonicalNames.NUMBER; + if (superTypes.contains(CanonicalNames.DATE)) return CanonicalNames.DATE; + } + return "OTHER"; + } + + + //used in Berant et al., 2013 and in the RL parser + //conjoins all binaries in the logical form with all non-entity lemmas void conjoinLemmaAndBinary(Example ex, Derivation deriv) { - if(!containsDomain("lemmaAndBinaries")) return; + if (!containsDomain("lemmaAndBinaries")) return; if (!deriv.isRoot(ex.numTokens())) return; - List nonEntityLemmas = new LinkedList(); - extractNonEntityLemmas(ex,deriv,nonEntityLemmas); + List nonEntityLemmas = new LinkedList<>(); + extractNonEntityLemmas(ex, deriv, nonEntityLemmas); List binaries = extractBinaries(deriv.formula); - if(opts.conjoinAllLemmas) { - deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas="+Joiner.on('_').join(nonEntityLemmas)+ - ",binaries="+Joiner.on('_').join(binaries)); - } - else { - String binariesStr = Joiner.on('_').join(binaries); - for(String nonEntityLemma: nonEntityLemmas) { - deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas="+nonEntityLemma+ - ",binaries="+binariesStr); - } + String binariesStr = Joiner.on('_').join(binaries); + for (String nonEntityLemma : nonEntityLemmas) { + deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas=" + nonEntityLemma + + ",binaries=" + binariesStr); } } -//TODO clean this + + // Extract the utterance that the derivation generates (not necessarily the + // one in the input utterance). + private void extractUtterance(Derivation deriv, List utterance) { + if (deriv.rule == Rule.nullRule) return; + int c = 0; // Index into children + for (String item : deriv.rule.rhs) { + if (Rule.isCat(item)) + extractUtterance(deriv.children.get(c++), utterance); + else + utterance.add(item); + } + } + + //Used in Berant et., EMNLP 2013, and in the agenda RL parser + //Extracts all content-word lemmas in the derivation tree not dominated by the category $Entity private void extractNonEntityLemmas(Example ex, Derivation deriv, List nonEntityLemmas) { - - - if(deriv.children.size()==0) {//base case this means it is a word that should be appended - for(int i = deriv.start; i < deriv.end; i++){ + if (deriv.children.size() == 0) { // base case this means it is a word that should be appended + for (int i = deriv.start; i < deriv.end; i++) { String pos = ex.languageInfo.posTags.get(i); - if((pos.startsWith("N") || pos.startsWith("V") || pos.startsWith("W") || pos.startsWith("A") || pos.equals("IN")) - && !ex.languageInfo.lemmaTokens.get(i).equals("be")) + if ((pos.startsWith("N") || pos.startsWith("V") || pos.startsWith("W") || pos.startsWith("A") || pos.equals("IN")) + && !ex.languageInfo.lemmaTokens.get(i).equals("be")) nonEntityLemmas.add(ex.languageInfo.lemmaTokens.get(i)); } - } - else { //recursion - for(Derivation child: deriv.children) { - if(child.rule.lhs == null || !child.rule.lhs.equals("$Entity")) { + } else { // recursion + for (Derivation child : deriv.children) { + if (child.rule.lhs == null || !child.rule.lhs.equals("$Entity")) { extractNonEntityLemmas(ex, child, nonEntityLemmas); - } - else if(child.rule.lhs.equals("$Entity")) { + } else if (child.rule.lhs.equals("$Entity")) { nonEntityLemmas.add("E"); } } } } -//TODO clean this + + //Used in Berant et al., 2013 and in agenda-based RL parser private List extractBinaries(Formula formula) { - List res = new LinkedList(); + List res = new LinkedList<>(); Set atomicElements = Formulas.extractAtomicFreebaseElements(formula); - for(String atomicElement: atomicElements) { - if(atomicElement.split("\\.").length==3 && !atomicElement.equals("fb:type.object.type")) + for (String atomicElement : atomicElements) { + if (atomicElement.split("\\.").length == 3 && !atomicElement.equals("fb:type.object.type")) res.add(atomicElement); } return res; } + + /** + * Add an indicator for each token alignment between a token in the utterance and its + * canonical form in the grammar using the ParaphraseModel, if applicable. + */ + // Compute paraphrasing features. In the future, need to store temporary + // state to make this more efficient. + void extractParaphraseFeatures(Example ex, Derivation deriv) { + if (!containsDomain("paraphrasing")) return; + if (deriv.rule == Rule.nullRule) return; + extractParaphrasePrecision(ex, deriv); + extractParaphraseRecall(ex, deriv); + + // Make sure we have a valid, floating, paraphraseable rule + if (!deriv.rule.isFloating() || !deriv.rule.isRhsTerminals()) return; + String key = join(deriv.rule.rhs, " "); + ParaphraseModel model = ParaphraseModel.getSingleton(); + if (!model.containsKey(key)) return; + for (String token : ex.getTokens()) { + double score = model.get(key, token); + if (score > 0.0) { + deriv.addFeature("paraphrasing", "(" + key + ", " + token + ")"); + } + } + } + + /** + * Add an indicator for each pair of bigrams that can be aligned from the original + * utterance and two (not necessarily contiguous) lemmas in the generated utterance + */ + private void extractBigramFeatures(Example ex, Derivation deriv) { + if (!containsDomain("bigram")) return; + if (!deriv.cat.equals(Rule.rootCat)) return; + LanguageInfo derivInfo = LanguageAnalyzer.getSingleton().analyze(deriv.canonicalUtterance); + List derivLemmas = derivInfo.lemmaTokens; + List exLemmas = ex.languageInfo.lemmaTokens; + for (int i = 0; i < exLemmas.size() - 1; i++) { + for (int j = 0; j < derivLemmas.size() - 1; j++) { + if (derivLemmas.get(j).equals(exLemmas.get(i))) { + // Consider bigrams separated by up to maxBigramDistance in generated utterance + for (int k = 1; j + k < derivLemmas.size() && k <= opts.maxBigramDistance; k++) { + if (derivLemmas.get(j + k).equals(exLemmas.get(i + 1))) { + deriv.addFeature("bigram", + exLemmas.get(i) + "," + exLemmas.get(i + 1) + " - " + k); + } + } + } + } + } + } + + private void extractParaphraseRecall(Example ex, Derivation deriv) { + if (!deriv.cat.equals(Rule.rootCat)) return; + String[] derivTokens = deriv.canonicalUtterance.split("\\s+"); + int n = 0; + for (int i = 0; i < ex.languageInfo.numTokens(); ++i) { + for (int j = 0; j < derivTokens.length; ++j) { + if (derivTokens[j].equals(ex.languageInfo.tokens.get(i))) { + n++; + break; + } + } + } + deriv.addFeature("paraphrasing", "deriv_recall", (double) n / ex.languageInfo.numTokens()); + } + + private void extractParaphrasePrecision(Example ex, Derivation deriv) { + //how many of the generated stuff is in the original utterance + int n = 0; + for (String item : deriv.rule.rhs) { + if (Rule.isCat(item)) continue; + if (ex.languageInfo.tokens.contains(item)) + n++; + } + deriv.addFeature("paraphrasing", "deriv_prec", n); + } + + // Joins arrayList of strings into string + String join(List l, String delimiter) { + StringBuilder sb = new StringBuilder(l.get(0)); + for (int i = 1; i < l.size(); i++) { + sb.append(delimiter); + sb.append(l.get(i)); + } + return sb.toString(); + } } diff --git a/src/edu/stanford/nlp/sempre/FeatureMatcher.java b/src/edu/stanford/nlp/sempre/FeatureMatcher.java index 897a75f..bcb9619 100644 --- a/src/edu/stanford/nlp/sempre/FeatureMatcher.java +++ b/src/edu/stanford/nlp/sempre/FeatureMatcher.java @@ -1,24 +1,24 @@ package edu.stanford.nlp.sempre; public interface FeatureMatcher { - public boolean matches(String feature); + boolean matches(String feature); } -class AllFeatureMatcher implements FeatureMatcher { +final class AllFeatureMatcher implements FeatureMatcher { private AllFeatureMatcher() { } @Override public boolean matches(String feature) { return true; } public static final AllFeatureMatcher matcher = new AllFeatureMatcher(); } -class ExactFeatureMatcher implements FeatureMatcher { +final class ExactFeatureMatcher implements FeatureMatcher { private String match; public ExactFeatureMatcher(String match) { this.match = match; } @Override public boolean matches(String feature) { return feature.equals(match); } } -class DenotationFeatureMatcher implements FeatureMatcher { +final class DenotationFeatureMatcher implements FeatureMatcher { @Override public boolean matches(String feature) { return feature.startsWith("denotation-size") || diff --git a/src/edu/stanford/nlp/sempre/FeatureVector.java b/src/edu/stanford/nlp/sempre/FeatureVector.java index 3e35cd2..d6e75c6 100644 --- a/src/edu/stanford/nlp/sempre/FeatureVector.java +++ b/src/edu/stanford/nlp/sempre/FeatureVector.java @@ -14,21 +14,22 @@ import java.util.*; * so that the key space isn't a free-for-all. * * @author Percy Liang + * @author Jonathan Berant */ public class FeatureVector { // These features map to the value 1 (most common case in NLP). private ArrayList indicatorFeatures; // General features private ArrayList> generalFeatures; - //A dense array of features to save memory + // A dense array of features to save memory private double[] denseFeatures; private static final String DENSE_NAME = "Dns"; - public FeatureVector() {} //constructor that does nothing + public FeatureVector() { } // constructor that does nothing public FeatureVector(int numOfDenseFeatures) { - denseFeatures=new double[numOfDenseFeatures]; - Arrays.fill(denseFeatures,0d); + denseFeatures = new double[numOfDenseFeatures]; + Arrays.fill(denseFeatures, 0d); } private static String toFeature(String domain, String name) { return domain + " :: " + name; } @@ -37,7 +38,7 @@ public class FeatureVector { add(toFeature(domain, name)); } private void add(String feature) { - if (indicatorFeatures == null) indicatorFeatures = new ArrayList(); + if (indicatorFeatures == null) indicatorFeatures = new ArrayList<>(); indicatorFeatures.add(feature); } @@ -45,7 +46,7 @@ public class FeatureVector { add(toFeature(domain, name), value); } private void add(String feature, double value) { - if (generalFeatures == null) generalFeatures = new ArrayList>(); + if (generalFeatures == null) generalFeatures = new ArrayList<>(); generalFeatures.add(Pair.newPair(feature, value)); } @@ -54,6 +55,30 @@ public class FeatureVector { add(domain, name + "-bias", 1); } + // Add histogram features, e.g., domain :: name>=4 + public void addHistogram(String domain, String name, double value) { addHistogram(domain, name, value, 2, 10, true); } + public void addHistogram(String domain, String name, double value, int initBinSize, int numBins, boolean exp) { + double upper = initBinSize; + String bin = null; + int sign = value > 0 ? +1 : -1; + value = Math.abs(value); + for (int i = 0; i < numBins; i++) { + double lastUpper = upper; + if (i > 0) { + if (exp) upper *= initBinSize; + else upper += initBinSize; + } + if (value < upper) { + bin = (sign > 0) ? lastUpper + ":" + upper : (-upper) + ":" + (-lastUpper); + break; + } + } + if (bin == null) + bin = (sign > 0) ? ">=" + upper : "<=" + (-upper); + + add(domain, name + bin); + } + public void addFromString(String feature, double value) { assert feature.contains(" :: ") : feature; if (value == 1) add(feature); @@ -61,7 +86,7 @@ public class FeatureVector { } public void addDenseFeature(int index, double value) { - denseFeatures[index]+=value; + denseFeatures[index] += value; } public void add(FeatureVector that) { add(that, AllFeatureMatcher.matcher); } @@ -76,11 +101,11 @@ public class FeatureVector { if (matcher.matches(pair.getFirst())) add(pair.getFirst(), pair.getSecond()); } - //dense features are always added - if(that.denseFeatures!=null) { - for(int i = 0; i < denseFeatures.length;++i) - denseFeatures[i]=that.denseFeatures[i]; - } + // dense features are always added + if (that.denseFeatures != null) { + for (int i = 0; i < denseFeatures.length; ++i) + denseFeatures[i] = that.denseFeatures[i]; + } } // Return the dot product between this feature vector and the weight vector (parameters). @@ -94,9 +119,9 @@ public class FeatureVector { for (Pair pair : generalFeatures) sum += params.getWeight(pair.getFirst()) * pair.getSecond(); } - if(denseFeatures != null) { - for(int i = 0; i < denseFeatures.length; ++i) - sum += params.getWeight(DENSE_NAME+"_"+i) * denseFeatures[i]; + if (denseFeatures != null) { + for (int i = 0; i < denseFeatures.length; ++i) + sum += params.getWeight(DENSE_NAME + "_" + i) * denseFeatures[i]; } return sum; } @@ -118,18 +143,33 @@ public class FeatureVector { MapUtils.incr(map, pair.getFirst(), factor * pair.getSecond()); } if (denseFeatures != null) { - for(int i = 0; i < denseFeatures.length;++i) - MapUtils.incr(map, DENSE_NAME+"_"+i, factor * denseFeatures[i]); + for (int i = 0; i < denseFeatures.length; ++i) + MapUtils.incr(map, DENSE_NAME + "_" + i, factor * denseFeatures[i]); } } + // returns a feature vector where all features are prefixed + public FeatureVector addPrefix(String prefix) { + FeatureVector res = new FeatureVector(); + if (indicatorFeatures != null) { + for (String feature : indicatorFeatures) + res.add(prefix + feature); + } + if (generalFeatures != null) { + for (Pair pair : generalFeatures) { + res.add(prefix + pair.getFirst(), pair.getSecond()); + } + } + return res; + } + @JsonValue public Map toMap() { HashMap map = new HashMap(); increment(1, map); - if(denseFeatures!=null) { - for(int i = 0; i < denseFeatures.length; ++i) { - map.put(DENSE_NAME+"_"+i,denseFeatures[i]); + if (denseFeatures != null) { + for (int i = 0; i < denseFeatures.length; ++i) { + map.put(DENSE_NAME + "_" + i, denseFeatures[i]); } } return map; @@ -140,22 +180,21 @@ public class FeatureVector { // TODO (rf): // Encoding is lossy. We guess that value of 1 means indicator, but we // could be wrong. - //(joberant) - takes care of dense features in a non efficient way - TODO + // TODO(joberant) - takes care of dense features in a non efficient way int maxDenseFeaturesIndex = -1; for (Map.Entry entry : m.entrySet()) { - if(isDenseFeature(entry.getKey())) { + if (isDenseFeature(entry.getKey())) { int index = denseFeatureIndex(entry.getKey()); - if(index>maxDenseFeaturesIndex) - maxDenseFeaturesIndex=index; + if (index > maxDenseFeaturesIndex) + maxDenseFeaturesIndex = index; } } - FeatureVector fv = maxDenseFeaturesIndex==-1 ? new FeatureVector() : new FeatureVector(maxDenseFeaturesIndex+1); + FeatureVector fv = maxDenseFeaturesIndex == -1 ? new FeatureVector() : new FeatureVector(maxDenseFeaturesIndex + 1); for (Map.Entry entry : m.entrySet()) { - if(isDenseFeature(entry.getKey())) { + if (isDenseFeature(entry.getKey())) { fv.addDenseFeature(denseFeatureIndex(entry.getKey()), entry.getValue()); - } - else { + } else { if (entry.getValue() == 1.0d) fv.add(entry.getKey()); else @@ -203,18 +242,18 @@ public class FeatureVector { } LogInfo.end_track(); } - + public static void logFeatures(Map features) { - for(String key: features.keySet()) { - LogInfo.logs("%s\t%s",key,features.get(key)); + for (String key : features.keySet()) { + LogInfo.logs("%s\t%s", key, features.get(key)); } } public void clear() { - if(indicatorFeatures!=null) + if (indicatorFeatures != null) indicatorFeatures.clear(); - if(generalFeatures!=null) + if (generalFeatures != null) generalFeatures.clear(); - denseFeatures=null; + denseFeatures = null; } } diff --git a/src/edu/stanford/nlp/sempre/FilterNerSpanFn.java b/src/edu/stanford/nlp/sempre/FilterNerSpanFn.java index 745d42e..4da87a6 100644 --- a/src/edu/stanford/nlp/sempre/FilterNerSpanFn.java +++ b/src/edu/stanford/nlp/sempre/FilterNerSpanFn.java @@ -3,7 +3,6 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -14,7 +13,7 @@ import java.util.List; */ public class FilterNerSpanFn extends SemanticFn { // Accepted NER tags (PERSON, LOCATION, ORGANIZATION, etc) - List acceptableNerTags = new ArrayList(); + List acceptableNerTags = new ArrayList<>(); public void init(LispTree tree) { super.init(tree); @@ -22,40 +21,42 @@ public class FilterNerSpanFn extends SemanticFn { acceptableNerTags.add(tree.child(j).value); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FilterNerSpanFn that = (FilterNerSpanFn) o; - if (!acceptableNerTags.equals(that.acceptableNerTags)) return false; - return true; + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + if (!isValid(ex, c)) + return null; + else { + return new Derivation.Builder() + .withCallable(c) + .withFormulaFrom(c.child(0)) + .createDerivation(); + } + } + }; } - public List call(Example ex, Callable c) { + private boolean isValid(Example ex, Callable c) { String nerTag = ex.languageInfo.nerTags.get(c.getStart()); // Check that it's an acceptable tag if (!acceptableNerTags.contains(nerTag)) - return Collections.emptyList(); + return false; // Check to make sure that all the tags are the same for (int j = c.getStart() + 1; j < c.getEnd(); j++) if (!nerTag.equals(ex.languageInfo.nerTags.get(j))) - return Collections.emptyList(); + return false; // Make sure that the whole NE is matched if (c.getStart() > 0 && nerTag.equals(ex.languageInfo.nerTags.get(c.getStart() - 1))) - return Collections.emptyList(); + return false; if (c.getEnd() < ex.languageInfo.nerTags.size() && - nerTag.equals(ex.languageInfo.nerTags.get(c.getEnd()))) - return Collections.emptyList(); - + nerTag.equals(ex.languageInfo.nerTags.get(c.getEnd()))) + return false; assert (c.getChildren().size() == 1) : c.getChildren(); - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .withFormulaFrom(c.child(0)) - .createDerivation()); + return true; } } diff --git a/src/edu/stanford/nlp/sempre/FilterPosTagFn.java b/src/edu/stanford/nlp/sempre/FilterPosTagFn.java index 13c867c..60de146 100644 --- a/src/edu/stanford/nlp/sempre/FilterPosTagFn.java +++ b/src/edu/stanford/nlp/sempre/FilterPosTagFn.java @@ -3,7 +3,6 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -14,8 +13,9 @@ import java.util.List; */ public class FilterPosTagFn extends SemanticFn { // Accepted POS tags (e.g., NNP, NNS, etc.) - List posTags = new ArrayList(); + List posTags = new ArrayList<>(); String mode; + boolean reverse; public void init(LispTree tree) { super.init(tree); @@ -23,62 +23,76 @@ public class FilterPosTagFn extends SemanticFn { if (!mode.equals("span") && !mode.equals("token")) throw new RuntimeException("Illegal description for whether to filter by token or span: " + tree.child(1).value); - for (int j = 2; j < tree.children.size(); j++) + for (int j = 2; j < tree.children.size(); j++) { + // Optionally, we can use a reverse filter (only reject certain tags) + if (j == 2 && tree.child(2).value.equals("reverse")) { + reverse = true; + continue; + } posTags.add(tree.child(j).value); + } } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FilterPosTagFn that = (FilterPosTagFn) o; - if (!mode.equals(that.mode)) return false; - if (!posTags.equals(that.posTags)) return false; - return true; + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + if (mode.equals("span")) + return callSpan(ex, c); + else + return callToken(ex, c); + } + }; } - public List call(Example ex, Callable c) { - if (mode.equals("span")) - return callSpan(ex, c); - return callToken(ex, c); - } - - private List callToken(Example ex, Callable c) { + private Derivation callToken(Example ex, Callable c) { // Only apply to single tokens - if (c.getEnd() - c.getStart() != 1) - return Collections.emptyList(); - String posTag = ex.posTag(c.getStart()); - if (posTags.contains(posTag)) { - return Collections.singletonList( - new Derivation.Builder() + if (c.getEnd() - c.getStart() != 1 || + (!posTags.contains(posTag) ^ reverse)) + return null; + else { + return new Derivation.Builder() .withCallable(c) .withFormulaFrom(c.child(0)) - .createDerivation()); + .createDerivation(); } - return Collections.emptyList(); } - private List callSpan(Example ex, Callable c) { + private Derivation callSpan(Example ex, Callable c) { + if (isValidSpan(ex, c)) { + return new Derivation.Builder() + .withCallable(c) + .withFormulaFrom(c.child(0)) + .createDerivation(); + } else { + return null; + } + } + + private boolean isValidSpan(Example ex, Callable c) { + if (reverse) { + for (int j = c.getStart(); j < c.getEnd(); j++) { + if (posTags.contains(ex.posTag(j))) + return false; + } + return true; + } String posTag = ex.posTag(c.getStart()); // Check that it's an acceptable tag - if (!posTags.contains(posTag)) return Collections.emptyList(); + if (!posTags.contains(posTag)) + return false; // Check to make sure that all the tags are the same for (int j = c.getStart() + 1; j < c.getEnd(); j++) { if (!posTag.equals(ex.posTag(j))) - return Collections.emptyList(); + return false; } // Make sure that the whole POS sequence is matched if (c.getStart() > 0 && posTag.equals(ex.posTag(c.getStart() - 1))) - return Collections.emptyList(); + return false; if (c.getEnd() < ex.numTokens() && posTag.equals(ex.posTag(c.getEnd()))) - return Collections.emptyList(); - + return false; assert (c.getChildren().size() == 1) : c.getChildren(); - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .withFormulaFrom(c.child(0)) - .createDerivation()); + return true; } } diff --git a/src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java b/src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java index 5fb8caa..d88d94d 100644 --- a/src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java +++ b/src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java @@ -2,36 +2,42 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; -import java.util.Collections; -import java.util.List; - public class FilterSpanLengthFn extends SemanticFn { private int minLength; + private int maxLength; + + private static final int NO_MAXIMUM = -1; + + public FilterSpanLengthFn() { } + public FilterSpanLengthFn(int minLength) { + init(LispTree.proto.newList("FilterSpanLengthFn", "" + minLength)); + } public void init(LispTree tree) { super.init(tree); minLength = Integer.parseInt(tree.child(1).value); + if (tree.children.size() > 2) { + maxLength = Integer.parseInt(tree.child(2).value); + } else { + maxLength = NO_MAXIMUM; + } } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FilterSpanLengthFn that = (FilterSpanLengthFn) o; - if (minLength != that.minLength) return false; - return true; + public DerivationStream call(Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + if (c.getEnd() - c.getStart() < minLength) + return null; + if (maxLength != NO_MAXIMUM && c.getEnd() - c.getStart() > maxLength) + return null; + return new Derivation.Builder() + .withCallable(c) + .withFormulaFrom(c.child(0)) + .createDerivation(); + } + }; } - - @Override - public List call(Example ex, Callable c) { - if (c.getEnd() - c.getStart() < minLength) - return Collections.emptyList(); - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .withFormulaFrom(c.child(0)) - .createDerivation()); - } - } diff --git a/src/edu/stanford/nlp/sempre/FloatingFeatureComputer.java b/src/edu/stanford/nlp/sempre/FloatingFeatureComputer.java new file mode 100644 index 0000000..da0f16c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/FloatingFeatureComputer.java @@ -0,0 +1,65 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; + +import fig.basic.IntPair; + +/** + * Extract features specific to the floating parser. + * + * @author ppasupat + */ +public class FloatingFeatureComputer implements FeatureComputer { + + @Override + public void extractLocal(Example ex, Derivation deriv) { + extractFloatingRuleFeatures(ex, deriv); + extractFloatingSkipFeatures(ex, deriv); + } + + // Conjunction of the rule and each lemma in the sentence + void extractFloatingRuleFeatures(Example ex, Derivation deriv) { + if (!FeatureExtractor.containsDomain("floatRule")) return; + for (String lemma : ex.getLemmaTokens()) + deriv.addFeature("floatRule", "lemma=" + lemma + ",rule=" + deriv.rule.toString()); + } + + // Look for words with no anchored rule applied + void extractFloatingSkipFeatures(Example ex, Derivation deriv) { + if (!FeatureExtractor.containsDomain("floatSkip")) return; + if (!deriv.isRoot(ex.numTokens())) return; + // Get all anchored tokens + boolean[] anchored = new boolean[ex.numTokens()]; + List stack = new ArrayList<>(); + stack.add(deriv); + while (!stack.isEmpty()) { + Derivation currentDeriv = stack.remove(stack.size() - 1); + if (deriv.start != -1) { + for (int i = deriv.start; i < deriv.end; i++) + anchored[i] = true; + } else { + for (Derivation child : currentDeriv.children) { + stack.add(child); + } + } + } + // Fire features based on tokens that are (not) anchored + // See if named entities are skipped + for (IntPair pair : ex.languageInfo.getNamedEntitySpans()) { + for (int i = pair.first; i < pair.second; i++) { + if (!anchored[i]) { + String nerTag = ex.languageInfo.nerTags.get(i); + deriv.addFeature("floatSkip", "skipped-ner=" + nerTag); + break; + } + } + } + // See which POS tags are skipped + for (int i = 0; i < anchored.length; i++) { + if (!anchored[i]) { + deriv.addFeature("floatSkip", "skipped-pos=" + ex.posTag(i)); + } + } + } + +} diff --git a/src/edu/stanford/nlp/sempre/FloatingParser.java b/src/edu/stanford/nlp/sempre/FloatingParser.java new file mode 100644 index 0000000..99cebf8 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/FloatingParser.java @@ -0,0 +1,357 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.*; + +import java.util.*; + +import static fig.basic.LogInfo.logs; + +/** + * A FloatingParser builds Derivations according to a Grammar without having to + * generate the input utterance. In contrast, a conventional chart parser (e.g., + * BeamParser) constructs parses for each span of the utterance. This is very + * inefficient when we're performing a more extractive semantic parsing task + * where many of the words are unaccounted for. + * + * Assume the Grammar is binarized and only has rules of the following form: + * $Cat => token + * $Cat => $Cat + * $Cat => token token + * $Cat => token $Cat + * $Cat => $Cat token + * $Cat => $Cat $Cat + * Each rule is either anchored or floating or both. + * + * Chart cells are either: + * - anchored: (cat, start, end) [these are effectively at depth 0] + * - floating: (cat, depth) [depends on anchored cells as base cases] + * + * Rules: + * cat => cat1 cat2 [binary] + * cat => cat1 [unary] + * Combinations: + * (cat1, start, end) => (cat, start, end) + * (cat1, depth) => (cat, depth) + * + * (cat1, start, mid), (cat2, mid, end) => (cat, start, end) + * (cat1, start, end), (cat2, depth) => (cat, depth + 1) + * (cat1, depth), (cat2, start, end) => (cat, depth + 1) + * (cat1, depth1), (cat2, depth2) => (cat, max(depth1, depth2)+1) + * + * @author Percy Liang + */ +public class FloatingParser extends Parser { + public static class Options { + @Option public int maxDepth = 10; + @Option public boolean defaultIsFloating = true; + @Option (gloss = "Flag specifying whether anchored spans/tokens can only be used once in a derivation") + public boolean useAnchorsOnce = false; + } + public static Options opts = new Options(); + + public FloatingParser(Spec spec) { super(spec); } + + public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) { + return new FloatingParserState(this, params, ex, computeExpectedCounts); + } +} + +/** + * Stores FloatingParser information about parsing a particular example. The actual + * parsing code lives here. + * + * Currently, many of the fields in ParserState are not used (chart). + * Those should be refactored out. + * + * @author Percy Liang + */ +class FloatingParserState extends ParserState { + // cell => list of derivations, formula set + // Examples of state: + // (category, depth) + // (category, depth, set of tokens) + private final Map> chart = new HashMap<>(); + + public FloatingParserState(FloatingParser parser, Params params, Example ex, boolean computeExpectedCounts) { + super(parser, params, ex, computeExpectedCounts); + } + + // Construct state names. + private Object floatingCell(String cat, int depth) { + return cat + ":" + depth; + } + private Object anchoredCell(String cat, int start, int end) { + return cat + "[" + start + "," + end + "]"; + } + private Object cell(String cat, int start, int end, int depth) { + return (start != -1) ? anchoredCell(cat, start, end) : floatingCell(cat, depth); + } + + private void addToChart(Object cell, Derivation deriv) { + if (Parser.opts.verbose >= 3) + LogInfo.logs("addToChart %s: %s", cell, deriv); + if (!deriv.isFeaturizedAndScored()) // A derivation could be belong in multiple cells. + featurizeAndScoreDerivation(deriv); + MapUtils.addToList(chart, cell, deriv); + } + + private void applyRule(Rule rule, int start, int end, int depth, Derivation child1, Derivation child2, String canonicalUtterance) { + if (Parser.opts.verbose >= 5) logs("applyRule %s [%s:%s] depth=%s, %s %s", rule, start, end, depth, child1, child2); + List children; + if (child1 == null) // 0-ary + children = Collections.emptyList(); + else if (child2 == null) // 1-ary + children = Collections.singletonList(child1); + else { + children = ListUtils.newList(child1, child2); + // optionally: ensure that specific anchors are only used once per final derivation + if (FloatingParser.opts.useAnchorsOnce && + FloatingRuleUtils.derivationAnchorsOverlap(child1, child2)) + return; + } + + DerivationStream results = rule.sem.call(ex, + new SemanticFn.CallInfo(rule.lhs, start, end, rule, children)); + while (results.hasNext()) { + Derivation newDeriv = results.next(); + newDeriv.canonicalUtterance = canonicalUtterance; + // Avoid repetitive floating cells + /* + if (depth > 0) { + if (MapUtils.getSet(setChart, rule.lhs).contains(newDeriv.formula)) { + continue; + } + if (rule.lhs == Rule.rootCat) + LogInfo.logs("adding %s to root for floating", newDeriv); + MapUtils.addToSet(setChart, rule.lhs, newDeriv.formula); + } + */ + addToChart(cell(rule.lhs, start, end, depth), newDeriv); + if (depth == -1) // In addition, anchored cells become floating at level 0 + addToChart(floatingCell(rule.lhs, 0), newDeriv); + } + } + + private void applyAnchoredRule(Rule rule, int start, int end, Derivation child1, Derivation child2, String canonicalUtterance) { + applyRule(rule, start, end, -1, child1, child2, canonicalUtterance); + } + + private void applyFloatingRule(Rule rule, int depth, Derivation child1, Derivation child2, String canonicalUtterance) { + applyRule(rule, -1, -1, depth, child1, child2, canonicalUtterance); + } + + private List getDerivations(Object cell) { + List derivations = chart.get(cell); + // logs("getDerivations %s => %s", cell, derivations); + if (derivations == null) return Derivation.emptyList; + return derivations; + } + + // Build derivations over span |start|, |end|. + private void buildAnchored(int start, int end) { + // Apply unary tokens on spans (rule $A (a)) + for (Rule rule : parser.grammar.rules) { + if (!rule.isAnchored()) continue; + if (rule.rhs.size() != 1 || rule.isCatUnary()) continue; + boolean match = (end - start == 1) && ex.token(start).equals(rule.rhs.get(0)); + if (match) + applyAnchoredRule(rule, start, end, null, null, rule.rhs.get(0)); + } + + // Apply binaries on spans (rule $A ($B $C)), ... + for (int mid = start + 1; mid < end; mid++) { + for (Rule rule : parser.grammar.rules) { + if (!rule.isAnchored()) continue; + if (rule.rhs.size() != 2) continue; + + String rhs1 = rule.rhs.get(0); + String rhs2 = rule.rhs.get(1); + boolean match1 = (mid - start == 1) && ex.token(start).equals(rhs1); + boolean match2 = (end - mid == 1) && ex.token(mid).equals(rhs2); + + if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) { // token $Cat + if (match1) { + List derivations = getDerivations(anchoredCell(rhs2, mid, end)); + for (Derivation deriv : derivations) + applyAnchoredRule(rule, start, end, deriv, null, rhs1 + " " + deriv.canonicalUtterance); + } + } else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // $Cat token + if (match2) { + List derivations = getDerivations(anchoredCell(rhs1, start, mid)); + for (Derivation deriv : derivations) + applyAnchoredRule(rule, start, end, deriv, null, deriv.canonicalUtterance + " " + rhs2); + } + } else if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // token token + if (match1 && match2) + applyAnchoredRule(rule, start, end, null, null, rhs1 + " " + rhs2); + } else { // $Cat $Cat + List derivations1 = getDerivations(anchoredCell(rhs1, start, mid)); + List derivations2 = getDerivations(anchoredCell(rhs2, mid, end)); + for (Derivation deriv1 : derivations1) + for (Derivation deriv2 : derivations2) + applyAnchoredRule(rule, start, end, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance); + } + } + } + + // Apply unary categories on spans (rule $A ($B)) + // Important: do this in topologically sorted order and after all the binaries are done. + for (Rule rule : parser.catUnaryRules) { + if (!rule.isAnchored()) continue; + List derivations = getDerivations(anchoredCell(rule.rhs.get(0), start, end)); + for (Derivation deriv : derivations) { + applyAnchoredRule(rule, start, end, deriv, null, deriv.canonicalUtterance); + } + } + } + + // Build floating derivations of exactly depth |depth|. + private void buildFloating(int depth) { + // Apply unary tokens on spans (rule $A (a)) + if (depth == 1) { + for (Rule rule : parser.grammar.rules) { + if (!rule.isFloating()) continue; + if (rule.rhs.size() != 1 || rule.isCatUnary()) continue; + applyFloatingRule(rule, depth, null, null, rule.rhs.get(0)); + } + } + + // Apply binaries on spans (rule $A ($B $C)), ... + for (Rule rule : parser.grammar.rules) { + if (!rule.isFloating()) continue; + if (rule.rhs.size() != 2) continue; + + String rhs1 = rule.rhs.get(0); + String rhs2 = rule.rhs.get(1); + + if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // token token + if (depth == 1) + applyFloatingRule(rule, depth, null, null, rhs1 + " " + rhs2); + } else if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) { // token $Cat + List derivations = getDerivations(floatingCell(rhs2, depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, rhs1 + " " + deriv.canonicalUtterance); + } else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // $Cat token + List derivations = getDerivations(floatingCell(rhs1, depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2); + } else { // $Cat $Cat + for (int subDepth = 0; subDepth < depth; subDepth++) { // depth-1 <=depth-1 + List derivations1 = getDerivations(floatingCell(rhs1, depth - 1)); + List derivations2 = getDerivations(floatingCell(rhs2, subDepth)); + for (Derivation deriv1 : derivations1) + for (Derivation deriv2 : derivations2) + applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance); + } + for (int subDepth = 0; subDepth < depth - 1; subDepth++) { // derivations1 = getDerivations(floatingCell(rhs1, subDepth)); + List derivations2 = getDerivations(floatingCell(rhs2, depth - 1)); + for (Derivation deriv1 : derivations1) + for (Derivation deriv2 : derivations2) + applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance); + } + } + } + + // Apply unary categories on spans (rule $A ($B)) + // Important: do this in topologically sorted order and after all the binaries are done. + for (Rule rule : parser.catUnaryRules) { + if (!rule.isFloating()) continue; + List derivations = getDerivations(floatingCell(rule.rhs.get(0), depth - 1)); + for (Derivation deriv : derivations) + applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance); + } + } + + void addToDerivations(Object cell, List derivations) { + List myDerivations = chart.get(cell); + if (myDerivations != null) + derivations.addAll(myDerivations); + } + + @Override public void infer() { + LogInfo.begin_track("FloatingParser.infer()"); + + // Base case ($TOKEN, $PHRASE) + for (Derivation deriv : gatherTokenAndPhraseDerivations()) { + addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv); + addToChart(floatingCell(deriv.cat, 0), deriv); + } + + Set categories = new HashSet<>(); + for (Rule rule : parser.grammar.rules) + categories.add(rule.lhs); + + // Build up anchored derivations (like the BeamParser) + int numTokens = ex.numTokens(); + for (int len = 1; len <= numTokens; len++) { + for (int i = 0; i + len <= numTokens; i++) { + buildAnchored(i, i + len); + for (String cat : categories) { + String cell = anchoredCell(cat, i, i + len).toString(); + /*if (cat.equals("$Word") || cat.equals("$Fragment")) { + LogInfo.logs("Looking at cell: %s", cell); + if (chart.get(cell) == null) LogInfo.logs("???"); + else { + for (Derivation deriv : chart.get(cell)) + LogInfo.logs("Found cell of interest: %s", deriv.toString()); + } + }*/ + pruneCell(cell, chart.get(cell)); + } + } + } + + // Build up floating derivations + for (int depth = 1; depth <= FloatingParser.opts.maxDepth; depth++) { + buildFloating(depth); + for (String cat : categories) { + String cell = floatingCell(cat, depth).toString(); + /*if (cat.equals("$Word") || cat.equals("$Fragment")) { + LogInfo.logs("Looking at floating cell: %s", cell); + if (chart.get(cell) == null) LogInfo.logs("???"); + else { + for (Derivation deriv : chart.get(cell)) + LogInfo.logs("Found cell of interest: %s", deriv.toString()); + } + }*/ + pruneCell(cell, chart.get(cell)); + } + } + + // Collect final predicted derivations + addToDerivations(anchoredCell(Rule.rootCat, 0, numTokens), predDerivations); + for (int depth = 1; depth <= FloatingParser.opts.maxDepth; depth++) + addToDerivations(floatingCell(Rule.rootCat, depth), predDerivations); + + // Compute gradient with respect to the predicted derivations + ensureExecuted(); + if (computeExpectedCounts) { + expectedCounts = new HashMap<>(); + ParserState.computeExpectedCounts(predDerivations, expectedCounts); + } + + // Example summary + if (Parser.opts.verbose >= 1) { + LogInfo.begin_track("Summary of Example %s", ex.getUtterance()); + for (Derivation deriv : predDerivations) + LogInfo.logs("Generated: canonicalUtterance=%s, value=%s", deriv.canonicalUtterance, deriv.value); + LogInfo.end_track(); + } + + LogInfo.end_track(); + } + + private void visualizeAnchoredChart(Set categories) { + for (String cat : categories) { + for (int len = 1; len <= numTokens; ++len) { + for (int i = 0; i + len <= numTokens; ++i) { + List derivations = getDerivations(anchoredCell(cat, i, i + len)); + for (Derivation deriv : derivations) { + LogInfo.logs("ParserState.visualize: %s(%s:%s): %s", cat, i, i + len, deriv); + } + } + } + } + } +} diff --git a/src/edu/stanford/nlp/sempre/FloatingRuleUtils.java b/src/edu/stanford/nlp/sempre/FloatingRuleUtils.java new file mode 100644 index 0000000..eed165d --- /dev/null +++ b/src/edu/stanford/nlp/sempre/FloatingRuleUtils.java @@ -0,0 +1,44 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; + +/** + * Utilities for floating rules. + */ +public final class FloatingRuleUtils { + private FloatingRuleUtils() { } // Should not be called. + + /** + * Get the anchored sub-derivations (sub-trees) of a derivation. + * I.e., gets all sub-derivations that are associated with a span of the utterance. + */ + public static List getDerivationAnchors(Derivation deriv) { + List anchors = new ArrayList<>(); + if (deriv.rule.isAnchored()) { + // if the sub-derivation is anchored to a span just add it + anchors.add(deriv); + } else if (!(deriv.children == null || deriv.children.size() == 0)) { + // if the derivation is not anchored but has children, recurse into children + for (Derivation child : deriv.children) + anchors.addAll(getDerivationAnchors(child)); + } + return anchors; + } + + /** + * Helper function to ensure that anchored spans are only used once in a final derivation. + * for example if A spans (or has a child that spans) [0, 3] and B spans (or has a child + * that spans) [2, 4] then we have an overlap. + */ + public static boolean derivationAnchorsOverlap(Derivation a, Derivation b) { + List aRoots = getDerivationAnchors(a); + List bRoots = getDerivationAnchors(b); + for (Derivation aRoot : aRoots) { + for (Derivation bRoot : bRoots) { + if (aRoot.start < bRoot.end && bRoot.start < aRoot.end) + return true; + } + } + return false; + } +} diff --git a/src/edu/stanford/nlp/sempre/Formula.java b/src/edu/stanford/nlp/sempre/Formula.java index 2d61da8..862150c 100644 --- a/src/edu/stanford/nlp/sempre/Formula.java +++ b/src/edu/stanford/nlp/sempre/Formula.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * A Formula is a logical form, which is the result of semantic parsing. Current * implementation is lambda calculus with primitives like description logic and @@ -16,8 +18,8 @@ import fig.basic.LispTree; * @author Percy Liang */ public abstract class Formula { - //cache the hashcode - private int hashCode=-1; + // cache the hashcode + private int hashCode = -1; // Serialize as LispTree. public abstract LispTree toLispTree(); @@ -25,6 +27,10 @@ public abstract class Formula { // Apply to formulas. If |func| returns null, then recurse on children. public abstract Formula map(Function func); + // Recursively perform some operation on each formula. + // Apply to formulas. If |func| returns an empty set or |alwaysRecurse|, then recurse on children. + public abstract List mapToList(Function> func, boolean alwaysRecurse); + @JsonValue public String toString() { return toLispTree().toString(); } @@ -33,14 +39,14 @@ public abstract class Formula { return Formulas.fromLispTree(LispTree.proto.parseFromString(str)); } - @Override abstract public boolean equals(Object o); + @Override public abstract boolean equals(Object o); @Override public int hashCode() { - if(hashCode==-1) + if (hashCode == -1) hashCode = computeHashCode(); return hashCode; } - - abstract public int computeHashCode(); + + public abstract int computeHashCode(); public static Formula nullFormula = new PrimitiveFormula() { public LispTree toLispTree() { return LispTree.proto.newLeaf("null"); } diff --git a/src/edu/stanford/nlp/sempre/FormulaGenerationInfo.java b/src/edu/stanford/nlp/sempre/FormulaGenerationInfo.java deleted file mode 100644 index 9157908..0000000 --- a/src/edu/stanford/nlp/sempre/FormulaGenerationInfo.java +++ /dev/null @@ -1,119 +0,0 @@ -package edu.stanford.nlp.sempre; - -import java.util.Comparator; - -import edu.stanford.nlp.sempre.FbFormulasInfo.BinaryFormulaInfo; -import edu.stanford.nlp.sempre.FbFormulasInfo.UnaryFormulaInfo; -import edu.stanford.nlp.sempre.FormulaRetriever.EntityInfo; -import edu.stanford.nlp.sempre.MergeFormula.Mode; -import fig.basic.LogInfo; - -public class FormulaGenerationInfo { - - public final UnaryFormulaInfo uInfo; - public final BinaryFormulaInfo bInfo; - public final BinaryFormulaInfo injectedInfo; - public final EntityInfo entityInfo1; - public final EntityInfo entityInfo2; - public final boolean isCount; - public final boolean isInject; - public final boolean isUnary; - - public FormulaGenerationInfo(BinaryFormulaInfo bInfo,BinaryFormulaInfo injectedInfo, EntityInfo entity1, - EntityInfo entity2, UnaryFormulaInfo uInfo, boolean isCount, boolean isInject, boolean isUnary) { - super(); - this.bInfo = bInfo; - this.injectedInfo=injectedInfo; - this.entityInfo1 = entity1; - this.entityInfo2 = entity2; - this.isCount = isCount; - this.isInject = isInject; - this.isUnary = isUnary; - this.uInfo = uInfo; - } - - public String getQuestionWord() { - return isCount? "how many" : "what"; - } - - /* - * Basically looks at all fields and constructs the formula accordingly - */ - public Formula generateFormula() { - - boolean generateAggregate=false; - if(isCount && - !(bInfo.expectedType1.equals(FreebaseInfo.INT) || - bInfo.expectedType1.equals(FreebaseInfo.FLOAT))) - generateAggregate=true; - //NO INJECTION - if(!generateAggregate && !isInject) { - if(isUnary) { - return new MergeFormula(MergeFormula.Mode.and, - uInfo.formula, new JoinFormula(bInfo.formula,entityInfo1.entity)); - } - else { - return new JoinFormula(bInfo.formula, entityInfo1.entity); - } - } - if(generateAggregate && !isInject) { - if(!isUnary) { - return new AggregateFormula(AggregateFormula.Mode.count, new JoinFormula(bInfo.formula, entityInfo1.entity)); - } - else { - return new AggregateFormula(AggregateFormula.Mode.count, new MergeFormula(MergeFormula.Mode.and, - uInfo.formula, new JoinFormula(bInfo.formula,entityInfo1.entity))); - } - } - //WITH INJECTION - if(isInject) { - JoinFormula bReduced = (JoinFormula) Formulas.betaReduction(new JoinFormula(bInfo.formula, entityInfo1.entity)); - JoinFormula injectionJoin = new JoinFormula(injectedInfo.formula,entityInfo2.entity); - Formula merge = new MergeFormula(Mode.and, bReduced.child, injectionJoin); - Formula injectedFormula; - if(isUnary) - injectedFormula = new MergeFormula(Mode.and,uInfo.formula,new JoinFormula(bReduced.relation, merge)); - else - injectedFormula = new JoinFormula(bReduced.relation, merge); - if(generateAggregate) - return new AggregateFormula(AggregateFormula.Mode.count,injectedFormula); - return injectedFormula; - } - //Should not reach this - throw new RuntimeException("Does not support formula generation from info="+this.toString()); - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("bInfo="+bInfo.formula+", "); - sb.append("eInfo1="+entityInfo1.entity+", "); - if(uInfo!=null) - sb.append("uInfo="+uInfo.formula+", "); - if(entityInfo2!=null) - sb.append("eInfo2="+entityInfo2.entity+", "); - sb.append("count="+isCount); - return sb.toString(); - } - - public void log() { - LogInfo.logs("FormulaGenerationInfo: uFormula=%s, bFormula=%s, injectedFormula=%s, eInfo1=%s, eInfo2=%s, isCount=%s, isInject=%s, isUnary=%s", - uInfo,bInfo,injectedInfo,entityInfo1,entityInfo2,isCount,isInject,isUnary); - } - - public class RetrievedFormulasComparator implements Comparator { - - @Override - public int compare(FormulaGenerationInfo o1, - FormulaGenerationInfo o2) { - if(o1.entityInfo1.popularity>o2.entityInfo1.popularity) - return -1; - if(o1.entityInfo1.popularityo2.bInfo.popularity) - return -1; - if(o1.bInfo.popularity retrieveFormulas(ParsingExample ex) { - - List res = new ArrayList(); - try { - boolean isCount = isCountUtterance(ex.utterance); - List> entities = getLexiconEntities(ex.languageInfo); - joinWithBinariesAndInject(ex.languageInfo,res, isCount, entities); - createUnaries(ex.languageInfo,res); - LogInfo.logs("FormulaRetriver.retrieveLexiconFromulas: number of formulas=%s",res.size()); - } catch (ParseException | IOException e) { - throw new RuntimeException(e); - } - return res; - } - - public List retrieveFormulas(Example ex) { - - LogInfo.begin_track("Retrieve formulas"); - List res = new ArrayList(); - try { - boolean isCount = isCountUtterance(ex.utterance); - List> entities = getLexiconEntities(ex.languageInfo); - joinWithBinariesAndInject(ex.languageInfo,res, isCount, entities); - createUnaries(ex.languageInfo,res); - LogInfo.logs("FormulaRetriver.retrieveLexiconFromulas: number of formulas=%s",res.size()); - } catch (ParseException | IOException e) { - throw new RuntimeException(e); - } - LogInfo.end_track(); - return res; - } - - private void joinWithBinariesAndInject(LanguageInfo lInfo, List res, boolean isCount, - List> entities) throws ParseException, IOException { - for(Pair spanAndEntryPair: entities) { - joinEntityWithBinariesAndInject(lInfo, res, isCount, entities, spanAndEntryPair); - } - } - - private void createUnaries(LanguageInfo lInfo, List fgInfos) { - - List toAdd = new ArrayList(); - for(FormulaGenerationInfo fgInfo: fgInfos) { - toAdd.addAll(createUnaries(lInfo, fgInfo)); - } - fgInfos.addAll(toAdd); - } - - private void joinEntityWithBinariesAndInject(LanguageInfo lInfo, List res, - boolean isCount, List> entities, Pair spanAndEntryPair) - throws ParseException, IOException { - EntityLexicalEntry entityEntry = spanAndEntryPair.getSecond(); - int binaryCounter=0; - for(String type: entityEntry.types) { - for(Formula binary: fbFormulasInfo.getBinariesForType2(type)) { - - BinaryFormulaInfo bInfo = fbFormulasInfo.getBinaryInfo(binary); - EntityInfo eInfo = new EntityInfo(entityEntry.fbDescriptions.iterator().next(),entityEntry.formula, entityEntry.popularity,spanAndEntryPair.getFirst()); - //hack to have less things - if(toFilter(bInfo,entityEntry)) - continue; - binaryCounter++; - - if(opts.verbose>=3) { - LogInfo.logs("FormulaRetriver.retrieveLexiconFormulas: " + - "text=%s, entity=%s, entityDesc=%s, entity popularity=%s, binary=%s, binary popularity=%s", - entityEntry.textDescription,entityEntry.formula,entityEntry.fbDescriptions,entityEntry.popularity, - binary,bInfo.popularity); - } - FormulaGenerationInfo fgInfo = new FormulaGenerationInfo(bInfo,null,eInfo,null, null, isCount, false,false); - res.add(fgInfo); - //now try to inject also - if(opts.createInjections) { - injectBinaries(lInfo,res,fgInfo,entities); - } - } - } - LogInfo.logs("Number of binaries for %s=%s", entityEntry.formula,binaryCounter); - } - - private List createUnaries(LanguageInfo lInfo, FormulaGenerationInfo inFgInfo) { - - List res = new ArrayList(); - Set subtypes = fbFormulasInfo.getSubtypesExclusive(inFgInfo.bInfo.expectedType1); - for(String subtype: subtypes) { - if(badDomain(subtype)) - continue; - Formula type1Formula = new JoinFormula(FreebaseInfo.TYPE, new ValueFormula(new NameValue(subtype))); - UnaryFormulaInfo uInfo = fbFormulasInfo.getUnaryInfo(type1Formula); - if(uInfo!=null) { - for(String description: uInfo.descriptions) { - if(validDescription(description)) { - List descriptionTokens = Arrays.asList(description.split("\\s+")); - IntPair unarySpan = getUnarySpan(lInfo); //where should we match the description - if(ParaphraseUtils.matchLists(lInfo.tokens.subList(unarySpan.first, unarySpan.second), descriptionTokens) || - ParaphraseUtils.matchLists(lInfo.lemmaTokens.subList(unarySpan.first, unarySpan.second), descriptionTokens)) { - FormulaGenerationInfo fInfo = - new FormulaGenerationInfo(inFgInfo.bInfo, inFgInfo.injectedInfo, inFgInfo.entityInfo1, inFgInfo.entityInfo2, uInfo, - inFgInfo.isCount,inFgInfo.isInject,true); - res.add(fInfo); - if(opts.verbose>=3) - fInfo.log(); - break; - } - } - } - } - } - return res; - } - - private IntPair getUnarySpan(LanguageInfo languageInfo) { - if(!(languageInfo.lemmaTokens.get(0).equals("what") || languageInfo.lemmaTokens.get(0).equals("which"))) - return new IntPair(); - int start=1, end=1; - for(; end < languageInfo.numTokens(); ++end) { - if(languageInfo.posTags.get(end).startsWith("V")) - break; - } - return new IntPair(start,end); - } - - private boolean validDescription(String description) { - if(description.equals("do") || description.equals("be") || description.equals("have")) - return false; - return true; - } - - private void injectBinaries(LanguageInfo lInfo, List res, FormulaGenerationInfo fgInfo, - List> entities) throws ParseException, IOException { - - if(!(fgInfo.bInfo.formula instanceof LambdaFormula)) return; - //1. Find the binaries that can be injected - List injections = fbFormulasInfo.getInjectableBinaries(fgInfo.bInfo.formula); - //2. For each one find the type2 and try to find entities - LogInfo.begin_track("Injecting %s injections to binary %s",injections.size(),fgInfo.bInfo.formula); - for(Formula injection: injections) { - BinaryFormulaInfo injectionInfo = fbFormulasInfo.getBinaryInfo(injection); - List injectedEntities = findInjectedEntities(lInfo,fgInfo.entityInfo1.span,injectionInfo.expectedType2,entities); - for(EntityInfo injectedEntity: injectedEntities) { - res.add(new FormulaGenerationInfo(fgInfo.bInfo, injectionInfo, fgInfo.entityInfo1, injectedEntity, fgInfo.uInfo, fgInfo.isCount, true, fgInfo.isUnary)); - } - } - LogInfo.end_track(); - } - - //hacky method - private List findInjectedEntities(LanguageInfo lInfo, - IntPair excludedSpan, String exType, List> exampleEntityEntries) throws ParseException, IOException { - - List res = new ArrayList(); - if(exType.equals(FreebaseInfo.DATE)) { - findInjectedTimeEntities(lInfo, excludedSpan, res); - } - else { - if(badDomain(exType)) return res; - if(opts.conservativeEntityExtraction) { //TODO try and simplify this block - Set maximalNonOverlappingSpans = ParaphraseUtils.getMaxNonOverlappingSpans(lInfo.getNamedEntitiesAndProperNouns()); //cache the NEs to save time - for(IntPair entitySpan: maximalNonOverlappingSpans) { - - if(ParaphraseUtils.intervalIntersect(entitySpan, excludedSpan) || - lInfo.nerTags.get(entitySpan.first).equals("DATE")) - continue; - - String entityTokens = lInfo.phrase(entitySpan.first, entitySpan.second); - String entityLemmas = lInfo.lemmaPhrase(entitySpan.first, entitySpan.second); - List entries = getEntityEntries(entityTokens); - - for(EntityLexicalEntry entry: entries) { - if(!entry.types.contains(exType)) - continue; - String entryDesc = entry.fbDescriptions.iterator().next(); - if(!entryDesc.equals(entityTokens) && - !entryDesc.equals(entityLemmas)) - continue; - res.add(new EntityInfo(entryDesc, entry.formula, entry.popularity,entitySpan)); - if(opts.verbose>=3) - LogInfo.logs("FormulaRetriver.findInjectedEntities: Adding injected entity=%s, description=%s, extype=%s",entry.formula,entryDesc,exType); - } - } - } - else { - for(Pair spanAndEntry: exampleEntityEntries) { - if(ParaphraseUtils.intervalIntersect(spanAndEntry.getFirst(), excludedSpan)) - continue; - EntityLexicalEntry entry = spanAndEntry.getSecond(); - if(spanAndEntry.getSecond().types.contains(exType)) { - String entryDesc = entry.fbDescriptions.iterator().next(); - res.add(new EntityInfo(entryDesc, entry.formula, entry.popularity,spanAndEntry.getFirst())); - } - } - } - } - return res; - } - - private void findInjectedTimeEntities(LanguageInfo lInfo, IntPair excludedSpan, - List res) { - for(int i = 0; i < lInfo.tokens.size(); i++) { - if(lInfo.isNumberAndDate(i)) { - IntPair span = new IntPair(i,i+1); - if(ParaphraseUtils.intervalIntersect(span, excludedSpan)) - continue; - String token = lInfo.tokens.get(i); - if(DateValue.parseDateValue(token).year!=-1) { - EntityInfo entityInfo = new EntityInfo(token, new ValueFormula(DateValue.parseDateValue(token)),0,span); - res.add(entityInfo); - if(opts.verbose>=3) - LogInfo.logs("FormulaRetiever.findInjectedEntities: Adding injected time entity=%s",entityInfo); - } - } - } - } - - private boolean toFilter(BinaryFormulaInfo bInfo, - EntityLexicalEntry entityEntry) { - - String binaryDesc = bInfo.formula.toString(); - String expectedType1 = bInfo.expectedType1; - if(removeEquivalents && fbFormulasInfo.hasOpposite(bInfo.formula)) { //we generate the equivalences in QuestionGenerator - if(!fbFormulasInfo.isReversed(bInfo.formula)) - return true; - } - if(badDomain(binaryDesc) || fbFormulasInfo.isCvt(expectedType1)) - return true; - if(opts.filterRelations && ParaphraseUtils.isInteger(entityEntry.textDescription)) - return true; - if(opts.filterRelations && FreebaseInfo.isPrimitive(expectedType1)) - return true; - return false; - } - - private boolean badDomain(String str) { - if(str.contains("fb:common.topic.alias")) - return false; - if(opts.filterRelations) { - return str.contains("fb:user.") || str.contains("fb:base.") || str.contains("fb:dataworld.") || - str.contains("fb:type.") || str.contains("fb:common.") || str.contains("fb:freebase."); - } - else { - return str.contains("fb:user.") || str.contains("fb:common."); - } - } - - private boolean isCountUtterance(String utterance) { - if(!opts.supportCountUtterances) - return false; - return utterance.startsWith("how many") || - utterance.startsWith("how much") || - utterance.startsWith("number of") || - utterance.startsWith("what is the number of"); - } - - private List> getLexiconEntities(LanguageInfo lInfo) throws ParseException, IOException { - - List> res = new ArrayList>(); - LogInfo.begin_track("Retrieving entities"); - if(opts.conservativeEntityExtraction) { //for webquestions - conservativeEntityExtraction(lInfo, res); - } - else { - allSpansEntityExtraction(lInfo, res); //for free917 - } - LogInfo.logs("number of entity entries=%s",res.size()); - LogInfo.end_track(); - return res; - } - - /* - * Go over all spans and get lexical entries - */ - private void allSpansEntityExtraction(LanguageInfo lInfo, - List> res) throws ParseException, - IOException { - for(int i = 0; i <= lInfo.tokens.size()-1; i++) { - for(int j = i+1; j <= lInfo.tokens.size(); j++) { - String entityDesc = lInfo.phrase(i, j); - String entityLemmas = lInfo.lemmaPhrase(i, j); - if(opts.verbose>=3) - LogInfo.logs("Retrieving: entry=%s",entityDesc); - List entries = getEntityEntries(entityDesc); - if(entries.isEmpty()) - entries = getEntityEntries(entityLemmas); - for(EntityLexicalEntry entry: entries) { - res.add(Pair.newPair(new IntPair(i, j), entry)); - } - } - } - } - - /** - * Generates entities conservatively, using 4 rules of backoff - * @param ex - * @param res - * @param excludedSpans - * @throws ParseException - * @throws IOException - */ - private void conservativeEntityExtraction(LanguageInfo lInfo, List> res) - throws ParseException, IOException { - - Set allEntitySpans = lInfo.getNamedEntitiesAndProperNouns(); - Set maximalNonOverlappingSpans = ParaphraseUtils.getMaxNonOverlappingSpans(allEntitySpans); - //first try to get exact match for maximal spans of named entities - for(IntPair maximalEntitySpan: maximalNonOverlappingSpans) { - String entityDesc = lInfo.phrase(maximalEntitySpan.first, maximalEntitySpan.second); - List entries = getEntityEntries(entityDesc); - for(EntityLexicalEntry entry: entries) - res.add(Pair.newPair(maximalEntitySpan, entry)); - } - //then try to get exact match for named entities (not maximal span) - if(res.isEmpty()) { - for(IntPair entitySpan: allEntitySpans) { - String entityDesc = lInfo.phrase(entitySpan.first, entitySpan.second); - List entries = getEntityEntries(entityDesc); - for(EntityLexicalEntry entry: entries) - res.add(Pair.newPair(entitySpan, entry)); - } - } - //if can't just go over NNPs and NE - if(res.isEmpty()) { - for(int i = 0; i < lInfo.numTokens(); i++) { - if(LanguageUtils.isEntity(lInfo, i)) { - List entries = getEntityEntries(lInfo.tokens.get(i)); - for(EntityLexicalEntry entry: entries) - res.add(Pair.newPair(new IntPair(i,i+1), entry)); - } - } - } - //if can't try all content words - if(res.isEmpty()) { - for(int i = 0; i < lInfo.numTokens(); i++) { - if(LanguageUtils.isContentWord(lInfo.getCanonicalPos(i))) { - List entries = getEntityEntries(lInfo.tokens.get(i)); - for(EntityLexicalEntry entry: entries) - res.add(Pair.newPair(new IntPair(i,i+1), entry)); - } - } - } - } - - @SuppressWarnings("unchecked") - private List getEntityEntries(String phrase) - throws IOException, ParseException { - List res = (List)lexicon.lookupEntities(phrase, Lexicon.opts.entitySearchStrategy); - return res.subList(0, Math.min(res.size(), opts.maxEntries)); //we do filtering here and not at lexicon so cache does not change - } - - /** - * Minimal information necessary for generating formula and extarcting features - * @author jonathanberant - */ - public class EntityInfo { - public final IntPair span; - public final String desc; - public final Formula entity; - public final double popularity; - - public EntityInfo(String description, Formula entity, double popularity, IntPair span) { - this.desc = description; - this.entity = entity; - this.popularity = popularity; - this.span = span; - } - - public String toString() { - return Joiner.on('\t').join(desc,entity,popularity); - } - } -} diff --git a/src/edu/stanford/nlp/sempre/Formulas.java b/src/edu/stanford/nlp/sempre/Formulas.java index 466a1f5..494c892 100644 --- a/src/edu/stanford/nlp/sempre/Formulas.java +++ b/src/edu/stanford/nlp/sempre/Formulas.java @@ -15,14 +15,15 @@ import java.util.Set; */ public abstract class Formulas { public static Formula fromLispTree(LispTree tree) { - // Primitive predicate - if (tree.isLeaf()) return new ValueFormula(new NameValue(tree.value, null)); + // Try to interpret as ValueFormula + if (tree.isLeaf()) // Leaves are name values + return new ValueFormula(new NameValue(tree.value, null)); + Value value = Values.fromLispTreeOrNull(tree); // General case + if (value != null) + return new ValueFormula(value); String func = tree.child(0).value; - if (func != null) { - if (func.equals("boolean") || func.equals("number") || func.equals("string") || func.equals("date")) - return new ValueFormula(Values.fromLispTree(tree)); if (func.equals("var")) return new VariableFormula(tree.child(1).value); if (func.equals("lambda")) @@ -42,7 +43,7 @@ public abstract class Formulas { } } - { // Merge: (intersect (fb:type.object.type fb:people.person) (fb:people.person.children fb:en.barack_obama)) + { // Merge: (and (fb:type.object.type fb:people.person) (fb:people.person.children fb:en.barack_obama)) MergeFormula.Mode mode = MergeFormula.parseMode(func); if (mode != null) return new MergeFormula(mode, fromLispTree(tree.child(1)), fromLispTree(tree.child(2))); @@ -57,21 +58,44 @@ public abstract class Formulas { { // Superlative: (argmax 1 1 (fb:type.object.type fb:people.person) (lambda x (!fb:people.person.height_meters (var x)))) SuperlativeFormula.Mode mode = SuperlativeFormula.parseMode(func); if (mode != null) { + Formula rank = parseIntToFormula(tree.child(1)); + Formula count = parseIntToFormula(tree.child(2)); return new SuperlativeFormula( mode, - Integer.parseInt(tree.child(1).value), - Integer.parseInt(tree.child(2).value), + rank, + count, fromLispTree(tree.child(3)), fromLispTree(tree.child(4))); } } + { // Arithmetic: (- (!fb:people.person.height_meters (var x)) (!fb:people.person.height_meters (var y))) + ArithmeticFormula.Mode mode = ArithmeticFormula.parseMode(func); + if (mode != null) + return new ArithmeticFormula(mode, fromLispTree(tree.child(1)), fromLispTree(tree.child(2))); + } + // Default is join: (fb:type.object.type fb:people.person) if (tree.children.size() != 2) throw new RuntimeException("Invalid number of arguments for join (want 2): " + tree); return new JoinFormula(fromLispTree(tree.child(0)), fromLispTree(tree.child(1))); } + // Special case to enable "argmax 1 1" rather than "argmax (number 1) (number 1)" + private static Formula parseIntToFormula(LispTree tree) { + try { + int i = Integer.parseInt(tree.value); + double d = (double) i; + NumberValue value = new NumberValue(d); + return new ValueFormula(value); + } catch (NumberFormatException e) { + Formula formula = fromLispTree(tree); + if (!(formula instanceof PrimitiveFormula)) + throw new RuntimeException("Rank and count of argmax must be variables or numbers"); + return formula; + } + } + // Replace occurrences of the variable reference |var| with |formula|. public static Formula substituteVar(Formula formula, final String var, final Formula replaceFormula) { return formula.map( @@ -89,6 +113,17 @@ public abstract class Formulas { }); } + // Replace top-level occurrences of |searchFormula| inside |formula| with |replaceFormula|. + public static Formula substituteFormula(Formula formula, final Formula searchFormula, final Formula replaceFormula) { + return formula.map( + new Function() { + public Formula apply(Formula formula) { + if (formula.equals(searchFormula)) return replaceFormula; + return null; + } + }); + } + // Beta-reduction. public static Formula lambdaApply(LambdaFormula func, Formula arg) { return substituteVar(func.body, func.var, arg); @@ -115,32 +150,35 @@ public abstract class Formulas { if (formula instanceof PrimitiveFormula) return formula.equals(var); if (formula instanceof MergeFormula) { - MergeFormula merge = (MergeFormula)formula; + MergeFormula merge = (MergeFormula) formula; return containsFreeVar(merge.child1, var) || containsFreeVar(merge.child2, var); } if (formula instanceof JoinFormula) { - JoinFormula join = (JoinFormula)formula; + JoinFormula join = (JoinFormula) formula; return containsFreeVar(join.relation, var) || containsFreeVar(join.child, var); } if (formula instanceof LambdaFormula) { - LambdaFormula lambda = (LambdaFormula)formula; + LambdaFormula lambda = (LambdaFormula) formula; if (lambda.var.equals(var.name)) return false; // Blocked by bound variable return containsFreeVar(lambda.body, var); } if (formula instanceof MarkFormula) { - MarkFormula mark = (MarkFormula)formula; + MarkFormula mark = (MarkFormula) formula; // Note: marks are transparent, unlike lambdas return containsFreeVar(mark.body, var); } if (formula instanceof ReverseFormula) { - return containsFreeVar(((ReverseFormula)formula).child, var); + return containsFreeVar(((ReverseFormula) formula).child, var); } if (formula instanceof AggregateFormula) { - return containsFreeVar(((AggregateFormula)formula).child, var); + return containsFreeVar(((AggregateFormula) formula).child, var); + } + if (formula instanceof ArithmeticFormula) { + return containsFreeVar(((ArithmeticFormula) formula).child1, var) || containsFreeVar(((ArithmeticFormula) formula).child2, var); } if (formula instanceof SuperlativeFormula) { - SuperlativeFormula superlative = (SuperlativeFormula)formula; - return containsFreeVar(superlative.head, var) || containsFreeVar(superlative.relation, var); + SuperlativeFormula superlative = (SuperlativeFormula) formula; + return containsFreeVar(superlative.rank, var) || containsFreeVar(superlative.count, var) || containsFreeVar(superlative.head, var) || containsFreeVar(superlative.relation, var); } if (formula instanceof NotFormula) { NotFormula notForm = (NotFormula) formula; @@ -148,32 +186,29 @@ public abstract class Formulas { } throw new RuntimeException("Unhandled: " + formula); } - - // TODO: use Formula.map + + // TODO(joberant): use Formula.map, and use CanonicalNames.isReverseProperty, etc. public static Set extractAtomicFreebaseElements(Formula formula) { - Set res = new HashSet(); + Set res = new HashSet<>(); LispTree formulaTree = formula.toLispTree(); extractAtomicFreebaseElements(formulaTree, res); return res; } private static void extractAtomicFreebaseElements(LispTree formulaTree, Set res) { - //base - if (formulaTree.isLeaf()) { + if (formulaTree.isLeaf()) { // base if (formulaTree.value.startsWith("fb:")) res.add(formulaTree.value); else if (formulaTree.value.startsWith("!fb:")) res.add(formulaTree.value.substring(1)); - } - //recursion - else { + } else { // recursion for (LispTree child : formulaTree.children) { extractAtomicFreebaseElements(child, res); } } } - // TODO: remove + // TODO(jonathan): move to feature extractor (this function doesn't seem fundamental) public static boolean isCountFormula(Formula formula) { if (formula instanceof AggregateFormula) return ((AggregateFormula) formula).mode == AggregateFormula.Mode.count; @@ -190,78 +225,143 @@ public abstract class Formulas { public static String getString(Formula formula) { if (formula instanceof ValueFormula) { - Value value = ((ValueFormula)formula).value; + Value value = ((ValueFormula) formula).value; if (value instanceof StringValue) - return ((StringValue)value).value; + return ((StringValue) value).value; + if (value instanceof NameValue) + return ((NameValue) value).id; + if (value instanceof NumberValue) + return ((NumberValue) value).value + ""; + } else if (formula instanceof VariableFormula) { + return ((VariableFormula) formula).name; } return null; } + public static String getNameId(Formula formula) { if (formula instanceof ValueFormula) { - Value value = ((ValueFormula)formula).value; + Value value = ((ValueFormula) formula).value; if (value instanceof NameValue) - return ((NameValue)value).id; + return ((NameValue) value).id; } return null; } + public static double getDouble(Formula formula) { + if (formula instanceof ValueFormula) { + Value value = ((ValueFormula) formula).value; + if (value instanceof NumberValue) + return ((NumberValue) value).value; + } + return Double.NaN; + } + + public static int getInt(Formula formula) { + return (int) getDouble(formula); + } + public static ValueFormula newNameFormula(String id) { return new ValueFormula(new NameValue(id)); } - + /* * Extract all subformulas in a string format (to also have primitive values) + * TODO(joberant): replace this with Formulas.map */ public static Set extractSubparts(Formula f) { Set res = new HashSet(); - extractSubpartsRecursive(f,res); + extractSubpartsRecursive(f, res); return res; } private static void extractSubpartsRecursive(Formula f, Set res) { - //base + // base res.add(f.toString()); - //recurse - if(f instanceof AggregateFormula) { + // recurse + if (f instanceof AggregateFormula) { AggregateFormula aggFormula = (AggregateFormula) f; extractSubpartsRecursive(aggFormula, res); - } - else if(f instanceof CallFormula) { + } else if (f instanceof CallFormula) { CallFormula callFormula = (CallFormula) f; - extractSubpartsRecursive(callFormula.func,res); - for(Formula argFormula: callFormula.args) - extractSubpartsRecursive(argFormula, res); - } - else if(f instanceof JoinFormula) { + extractSubpartsRecursive(callFormula.func, res); + for (Formula argFormula : callFormula.args) + extractSubpartsRecursive(argFormula, res); + } else if (f instanceof JoinFormula) { JoinFormula joinFormula = (JoinFormula) f; - extractSubpartsRecursive(joinFormula.relation,res); - extractSubpartsRecursive(joinFormula.child,res); - } - else if(f instanceof LambdaFormula) { + extractSubpartsRecursive(joinFormula.relation, res); + extractSubpartsRecursive(joinFormula.child, res); + } else if (f instanceof LambdaFormula) { LambdaFormula lambdaFormula = (LambdaFormula) f; extractSubpartsRecursive(lambdaFormula.body, res); - } - else if(f instanceof MarkFormula) { + } else if (f instanceof MarkFormula) { MarkFormula markFormula = (MarkFormula) f; extractSubpartsRecursive(markFormula.body, res); - } - else if(f instanceof MergeFormula) { + } else if (f instanceof MergeFormula) { MergeFormula mergeFormula = (MergeFormula) f; extractSubpartsRecursive(mergeFormula.child1, res); extractSubpartsRecursive(mergeFormula.child2, res); - } - else if(f instanceof NotFormula) { + } else if (f instanceof NotFormula) { NotFormula notFormula = (NotFormula) f; extractSubpartsRecursive(notFormula.child, res); - } - else if(f instanceof ReverseFormula) { + } else if (f instanceof ReverseFormula) { ReverseFormula revFormula = (ReverseFormula) f; - extractSubpartsRecursive(revFormula.child,res); - } - else if(f instanceof SuperlativeFormula) { + extractSubpartsRecursive(revFormula.child, res); + } else if (f instanceof SuperlativeFormula) { SuperlativeFormula superlativeFormula = (SuperlativeFormula) f; + extractSubpartsRecursive(superlativeFormula.rank, res); + extractSubpartsRecursive(superlativeFormula.count, res); extractSubpartsRecursive(superlativeFormula.head, res); extractSubpartsRecursive(superlativeFormula.relation, res); } } + + // Takes in a |rawFormula| which represents a function x => y and returns a + // function y => x. + public static Formula reverseFormula(Formula rawFormula) { + if (rawFormula instanceof ValueFormula) { + @SuppressWarnings({ "unchecked" }) + ValueFormula vf = (ValueFormula) rawFormula; + return reverseNameFormula(vf); + } else if (rawFormula instanceof LambdaFormula) { + // Convert (lambda x (relation1 (relation2 (var x)))) <=> (lambda x (!relation2 (!relation1 (var x)))) + // Note: currently only handles chains. Make this more generic. + LambdaFormula formula = (LambdaFormula) rawFormula; + if (isChain(formula.body)) + return new LambdaFormula(formula.var, reverseChain(formula.body, new VariableFormula(formula.var))); + else + return new ReverseFormula(formula); + } else { + return new ReverseFormula(rawFormula); + // throw new RuntimeException("Not handled: " + rawFormula); + } + } + + // Helper function for reverseFormula(). + // Check to see if formula has the form (a (b (c (var x)))) + private static boolean isChain(Formula source) { + if (source instanceof JoinFormula) { + JoinFormula join = (JoinFormula) source; + return isChain(join.child); + } + return source instanceof VariableFormula; + } + // Reverse the chain + private static Formula reverseChain(Formula source, Formula result) { + if (source instanceof JoinFormula) { + JoinFormula join = (JoinFormula) source; + return reverseChain(join.child, new JoinFormula(reverseFormula(join.relation), result)); + } else if (source instanceof VariableFormula) { + return result; + } else { + throw new RuntimeException("Not handled: " + source); + } + } + + // !fb:people.person.place_of_birth <=> fb:people.person.place_of_birth + private static ValueFormula reverseNameFormula(ValueFormula formula) { + String id = formula.value.id; + return new ValueFormula<>( + new NameValue(CanonicalNames.isReverseProperty(id) ? id.substring(1) : "!" + id)); + } + } diff --git a/src/edu/stanford/nlp/sempre/FreebaseInfo.java b/src/edu/stanford/nlp/sempre/FreebaseInfo.java deleted file mode 100644 index b75574c..0000000 --- a/src/edu/stanford/nlp/sempre/FreebaseInfo.java +++ /dev/null @@ -1,364 +0,0 @@ -package edu.stanford.nlp.sempre; - -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import edu.stanford.nlp.sempre.FbFormulasInfo.BinaryFormulaInfo; -import edu.stanford.nlp.sempre.FbFormulasInfo.UnaryFormulaInfo; -import fig.basic.*; - -import java.io.BufferedReader; -import java.io.IOException; -import java.util.*; - -/** - * Class for keeping info from Freebase schema - * @author jonathanberant - */ -public class FreebaseInfo { - - private static FreebaseInfo FreebaseInfo; - public static FreebaseInfo getSingleton() { - if (FreebaseInfo == null) FreebaseInfo = new FreebaseInfo(); - return FreebaseInfo; - } - - public static class Options { - @Option(gloss = "ttl file with schema information") - public String schemaPath = "lib/fb_data/93.exec/schema.ttl"; - } - public static Options opts = new Options(); - - // Types: number (boolean, int, float), date, text, entity (people, loc, org, ...), cvt - - // Concrete primitive types - public final static String BOOLEAN = "fb:type.boolean"; - public final static String INT = "fb:type.int"; - public final static String FLOAT = "fb:type.float"; - public final static String DATE = "fb:type.datetime"; - public final static String TEXT = "fb:type.text"; - - // Entity types - public final static String PERSON = "fb:people.person"; - public final static String LOC = "fb:location.location"; - public final static String ORG = "fb:organization.organization"; - - // Non-standard abstract types - public final static String NUMBER = "fb:type.number"; - public final static String ENTITY = "fb:type.entity"; - public final static String CVT = "fb:type.cvt"; - - // Common relations - public final static String TYPE = "fb:type.object.type"; - public final static String PROF = "fb:people.person.profession"; - public final static String NAME = "fb:type.object.name"; - public final static String ALIAS = "fb:common.topic.alias"; - - // Transitive types - public final static String CONTAINED_BY = "fb:location.location.containedby"; - - private BiMap masterToReverseMap = HashBiMap.create(); //mapping from master property to its reverse - private Map> typeToIncludedTypesMap = new HashMap>(); - private Map> typeToSubTypesMap = new HashMap>(); - private Set cvts = new HashSet(); - private Map type1Map = new HashMap(); // property => type of arg1 - private Map type2Map = new HashMap(); // property => type of arg2 - private Map unit2Map = new HashMap(); // property => unit of arg2 (if exists) - private Map> bDescriptionsMap = new HashMap>(); //property => descriptions - private Map bPopularityMap = new HashMap(); //property => popularity - //unary maps - private Map professionPopularityMap = new HashMap(); //property => popularity - private Map typePopularityMap = new HashMap(); //property => popularity - private Map> professionDescriptionsMap = new HashMap>(); //property => descriptions - private Map> typeDescriptionsMap = new HashMap>(); //property => descriptions - - private FreebaseInfo() { - try { - readSchema(); - } catch (NumberFormatException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - // For each type, add |type| and common topic to the set of supertypes. - for (Map.Entry> e : typeToIncludedTypesMap.entrySet()) - addDefaultSupertypes(e.getKey(), e.getValue()); - - // Non common topic supertypes. - addSupertype(INT, INT); - addSupertype(INT, NUMBER); - addSupertype(FLOAT, FLOAT); - addSupertype(FLOAT, NUMBER); - addSupertype(DATE, DATE); - } - - /** - * Go over schema twice - once to populate all fields except descriptions, the second time we populate descriptions after we now what - * are the properties we are interested in - * @throws NumberFormatException - * @throws IOException - */ - public void readSchema() throws NumberFormatException, IOException { - - LogInfo.begin_track("Loading Freebase schema: %s", opts.schemaPath); - BufferedReader in = IOUtils.openInHard(opts.schemaPath); - - String line; - while ((line = in.readLine()) != null) { - String[] tokens = edu.stanford.nlp.sempre.freebase.Utils.parseTriple(line); - if (tokens == null) continue; - String arg1 = tokens[0]; - String property = tokens[1]; - String arg2 = tokens[2]; - - if (property.equals("fb:type.property.reverse_property")) { - // Duplicates logically really shouldn't happen but the Freebase RDF - // reverse properties are not 1:1. We should monitor this and make - // sure we don't lose any alignments. - if (masterToReverseMap.containsKey(arg1)) { - //LogInfo.errors("arg1 exists multiple times: %s", line); - continue; - } - if (masterToReverseMap.inverse().containsKey(arg2)) { - //LogInfo.errors("arg2 exists multiple times: %s", line); - continue; - } - masterToReverseMap.put(arg1, arg2); - } else if (property.equals("fb:freebase.type_hints.included_types")) { - Set set = typeToIncludedTypesMap.get(arg1); - if (set == null) { - typeToIncludedTypesMap.put(arg1, set = new HashSet()); - } - set.add(arg2); - set = typeToSubTypesMap.get(arg2); - if (set == null) { - typeToSubTypesMap.put(arg2, set = new HashSet()); - } - set.add(arg1); - } else if (property.equals("fb:freebase.type_hints.mediator")) { - if (arg2.equals("\"true\"^^xsd:boolean")) cvts.add(arg1); - else if (arg2.equals("\"false\"^^xsd:boolean")) cvts.remove(arg1); - else throw new RuntimeException("Invalid xsd:boolean: " + arg2); - } else if (property.equals("fb:type.property.schema")) { - type1Map.put(arg1, arg2); - } else if (property.equals("fb:type.property.expected_type")) { - type2Map.put(arg1, arg2); - } else if (property.equals("fb:type.property.unit")) { - unit2Map.put(arg1, arg2); - } else if (property.equals("fb:user.custom.type.property.num_instances")) { - bPopularityMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseInt(arg2)); - } else if (property.equals("fb:user.custom.people.person.profession.num_instances")) { - professionPopularityMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseInt(arg2)); - } else if (property.equals("fb:user.custom.type.object.type.num_instances")) { - typePopularityMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseInt(arg2)); - } - } - in.close(); - //second iteration - populate descriptions assumes all properties have the fb:type.property.num_instances field - in = IOUtils.openInHard(opts.schemaPath); - while ((line = in.readLine()) != null) { - String[] tokens = edu.stanford.nlp.sempre.freebase.Utils.parseTriple(line); - if (tokens == null) continue; - String arg1 = tokens[0]; - String property = tokens[1]; - String arg2 = tokens[2]; - - if(property.equals(NAME) || property.equals(ALIAS)) { - if(bPopularityMap.containsKey(arg1)) { - MapUtils.addToList(bDescriptionsMap, arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2).toLowerCase()); - } - else if(professionPopularityMap.containsKey(arg1)) { - MapUtils.addToList(professionDescriptionsMap, arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2).toLowerCase()); - } - else if(typePopularityMap.containsKey(arg1)) { - MapUtils.addToList(typeDescriptionsMap, arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2).toLowerCase()); - } - } - } - LogInfo.logs("%d CVTs, (%d,%d) property types, %d property units", cvts.size(), type1Map.size(), type2Map.size(), unit2Map.size()); - LogInfo.end_track(); - } - - public Map createBinaryFormulaInfoMap() { - - Map res = new HashMap(); - for(String property: bPopularityMap.keySet()) { - Formula f = Formulas.fromLispTree(LispTree.proto.parseFromString(property)); - BinaryFormulaInfo info = new BinaryFormulaInfo(f, type1Map.get(property), type2Map.get(property), unit2Map.get(property),"",bDescriptionsMap.get(property),bPopularityMap.get(property)); - if(!info.isComplete()) { - continue; - } - res.put(f, info); - } - return res; - } - - public Map createUnaryFormulaInfoMap() { - - Map res = new HashMap(); - //professions - for(String profession: professionPopularityMap.keySet()) { - Formula f = new JoinFormula(PROF, new ValueFormula(new NameValue(profession))); - UnaryFormulaInfo info = new UnaryFormulaInfo(f, professionPopularityMap.get(profession), - MapUtils.get(professionDescriptionsMap,profession,new LinkedList()), - Collections.singleton(PERSON)); - if(!info.isComplete()) { - continue; - } - res.put(f, info); - } - //types - for(String type: typePopularityMap.keySet()) { - Formula f = new JoinFormula(TYPE, new ValueFormula(new NameValue(type))); - UnaryFormulaInfo info = new UnaryFormulaInfo(f, typePopularityMap.get(type), - MapUtils.get(typeDescriptionsMap,type,new LinkedList()), - Collections.singleton(type)); - if(!info.isComplete()) { - continue; - } - res.put(f, info); - } - return res; - } - - public boolean fbPropertyHasOpposite(String fbProperty) { - return masterToReverseMap.containsKey(fbProperty) || masterToReverseMap.inverse().containsKey(fbProperty); - } - //check if has opposite before using - public String getOppositeFbProperty(String fbPropety) { - if (masterToReverseMap.containsKey(fbPropety)) - return masterToReverseMap.get(fbPropety); - if (masterToReverseMap.inverse().containsKey(fbPropety)) - return masterToReverseMap.inverse().get(fbPropety); - throw new RuntimeException("Property does not have an opposite: " + fbPropety); - } - - private Set addDefaultSupertypes(String type, Set supertypes) { - supertypes.add(type); - supertypes.add("fb:common.topic"); - return supertypes; - } - - public void addSupertype(String subtype, String supertype) { - Set supertypes = typeToIncludedTypesMap.get(subtype); - if (supertypes == null) - typeToIncludedTypesMap.put(subtype, supertypes = new HashSet()); - supertypes.add(supertype); - - Set subTypes = typeToSubTypesMap.get(supertype); - if (subTypes == null) - typeToSubTypesMap.put(supertype, subTypes = new HashSet()); - subTypes.add(subtype); - } - - //Get the measurement unit associated with arg2 of property. - // If something is not a number, then return something crude (e.g. fb:type.cvt). - // Return null if we don't know anything. - public String getUnit2(String property) { - String type = type2Map.get(property); - if (type == null) { - //LogInfo.errors("No type information for property: %s", property); - return null; - } - if (type.equals(INT) || type.equals(FLOAT)) { - String unit = unit2Map.get(property); - if (unit == null) { - //LogInfo.errors("No unit information for property: %s", property); - return NumberValue.unitless; - } - return unit; - } - if (type.equals(BOOLEAN) || type.equals(TEXT) || type.equals(DATE)) // Use the type as the unit - return type; - if (isCvt(type)) return CVT; // CVT - return ENTITY; // Entity - } - - public boolean isCvt(String type) { - return cvts.contains(type); - } - - /* - public void computeTransitiveClosureInefficiently() { - - boolean added; - do { - added = false; - for (String subType : typeToIncludedTypesMap.keySet()) { - - Set superTypes = typeToIncludedTypesMap.get(subType); - Set typesToAdd = new HashSet(); - - for (String superType : superTypes) { - Set superSuperTypes = typeToIncludedTypesMap.get(superType); - if (superSuperTypes != null) { - typesToAdd.addAll(superSuperTypes); - } - } - typesToAdd.removeAll(superTypes); - if (typesToAdd.size() > 0) { - LogInfo.log("Adding to subtype: " + subType + "with supertypes " + superTypes + " the new types: " + typesToAdd); - superTypes.addAll(typesToAdd); - added = true; - } - } - } while (added == true); - }*/ - - public Set getIncludedTypesInclusive(String subtype) { - Set set = typeToIncludedTypesMap.get(subtype); - if (set == null) { - return addDefaultSupertypes(subtype, new HashSet()); - } - return set; - } - - public Set getSubTypesExclusive(String subtype) { - Set set = typeToSubTypesMap.get(subtype); - if (set == null) { - return new HashSet(); - } - return set; - } - - public String coarseType(String type) { - Set superTypes = typeToIncludedTypesMap.get(type); - if (superTypes != null) { - if (superTypes.contains(PERSON)) return PERSON; - if (superTypes.contains(LOC)) return LOC; - if (superTypes.contains(ORG)) return ORG; - if (superTypes.contains(NUMBER)) return NUMBER; - if (superTypes.contains(DATE)) return DATE; - } - return "OTHER"; // Shouldn't really happen - } - - // Return whether |property| is the name of a reverse property. - // Convention: ! is the prefix for reverses. - public static boolean isReverseProperty(String property) { - return property.startsWith("!") && !property.equals("!="); - } - - // fb:en.barack_obama => http://rdf.freebase.com/ns/en/barack_obama - public static final String freebaseNamespace = "http://rdf.freebase.com/ns/"; - - public static String id2uri(String id) { - assert id.startsWith("fb:") : id; - return freebaseNamespace + id.substring(3).replaceAll("\\.", "/"); - } - public static String uri2id(String uri) { - if (!uri.startsWith(freebaseNamespace)) { - LogInfo.logs("Warning: invalid Freebase uri: %s", uri); - // Don't do any conversion; this is not necessarily the best thing to do. - return uri; - } - return "fb:" + uri.substring(freebaseNamespace.length()).replaceAll("/", "."); - } - - public static boolean isPrimitive(String type) { - return type.equals(BOOLEAN) || - type.equals(INT) || - type.equals(FLOAT) || - type.equals(DATE) || - type.equals(TEXT); - } -} diff --git a/src/edu/stanford/nlp/sempre/FuncSemType.java b/src/edu/stanford/nlp/sempre/FuncSemType.java new file mode 100644 index 0000000..cebfa40 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/FuncSemType.java @@ -0,0 +1,47 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LispTree; + +/** + * FuncSemType really is used to represent a pair type (t1, t2) (despite its name). + * The lisp tree representation is (-> retType argType). + */ +public class FuncSemType extends SemType { + public final SemType argType; + public final SemType retType; + public FuncSemType(SemType argType, SemType retType) { + if (argType == null) throw new RuntimeException("Null argType"); + if (retType == null) throw new RuntimeException("Null retType"); + this.argType = argType; + this.retType = retType; + } + public FuncSemType(String argType, String retType) { + this(new AtomicSemType(argType), new AtomicSemType(retType)); + } + public boolean isValid() { return true; } + + public SemType meet(SemType that) { + if (that instanceof TopSemType) return this; + if (!(that instanceof FuncSemType)) return SemType.bottomType; + // Perform the meet elementwise (remember, treat this as a pair type). + FuncSemType thatFunc = (FuncSemType) that; + SemType newArgType = argType.meet(thatFunc.argType); + if (!newArgType.isValid()) return SemType.bottomType; + SemType newRetType = retType.meet(thatFunc.retType); + if (!newRetType.isValid()) return SemType.bottomType; + return new FuncSemType(newArgType, newRetType); + } + + public SemType apply(SemType that) { + if (argType.meet(that).isValid()) return retType; + return SemType.bottomType; + } + public FuncSemType reverse() { return new FuncSemType(retType, argType); } + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild("->"); + tree.addChild(argType.toLispTree()); + tree.addChild(retType.toLispTree()); + return tree; + } +} diff --git a/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java b/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java new file mode 100644 index 0000000..9d4efb1 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/FuzzyMatchFn.java @@ -0,0 +1,102 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; + +import fig.basic.*; + +/** + * Similar to LexiconFn, but list all approximate matches from a TableKnowledgeGraph. + * + * @author ppasupat + */ +public class FuzzyMatchFn extends SemanticFn { + public static class Options { + @Option public int verbose = 0; + } + public static Options opts = new Options(); + + public enum FuzzyMatchFnMode { UNARY, BINARY, ENTITY }; + private FuzzyMatchFnMode mode; + + // Generate all possible denotations regardless of the phrase + private boolean matchAny = false; + + public void init(LispTree tree) { + super.init(tree); + for (int i = 1; i < tree.children.size(); i++) { + String value = tree.child(i).value; + if ("unary".equals(value)) this.mode = FuzzyMatchFnMode.UNARY; + else if ("binary".equals(value)) this.mode = FuzzyMatchFnMode.BINARY; + else if ("entity".equals(value)) this.mode = FuzzyMatchFnMode.ENTITY; + else if ("any".equals(value)) this.matchAny = true; + else throw new RuntimeException("Invalid argument: " + value); + } + } + + @Override + public DerivationStream call(Example ex, Callable c) { + return new LazyFuzzyMatchFnDerivs(ex, c, mode, matchAny); + } + + // ============================================================ + // Derivation Stream + // ============================================================ + + public static class LazyFuzzyMatchFnDerivs extends MultipleDerivationStream { + final Example ex; + final KnowledgeGraph graph; + final Callable c; + final String query; + final FuzzyMatchFnMode mode; + final boolean matchAny; + + int index = 0; + List formulas; + + public LazyFuzzyMatchFnDerivs(Example ex, Callable c, FuzzyMatchFnMode mode, boolean matchAny) { + this.ex = ex; + this.graph = (ex.context == null) ? null : ex.context.graph; + this.c = c; + this.query = matchAny ? null : c.childStringValue(0); + this.mode = mode; + this.matchAny = matchAny; + if (opts.verbose >= 2) + LogInfo.logs("FuzzyMatchFn[%s]%s.call: %s", + this.mode, (this.matchAny ? "[matchAny]" : ""), this.query); + } + + @Override + public Derivation createDerivation() { + if (graph == null) return null; + + // Compute the formulas if not computed yet + if (formulas == null) { + if (matchAny) + formulas = new ArrayList<>(graph.getAllFormulas(mode)); + else + formulas = new ArrayList<>(graph.getFuzzyMatchedFormulas(query, mode)); + } + + // Use the next formula to create a derivation + if (index >= formulas.size()) return null; + Formula formula = formulas.get(index++); + SemType type = TypeInference.inferType(formula); + + FeatureVector features = new FeatureVector(); + if (FeatureExtractor.containsDomain("fuzzyMatch")) { + features.add("fuzzyMatch", "mode=" + mode); + if (matchAny) + features.add("fuzzyMatch", "mode=" + mode + "_any"); + } + + return new Derivation.Builder() + .withCallable(c) + .formula(formula) + .type(type) + .localFeatureVector(features) + .createDerivation(); + } + + } + +} diff --git a/src/edu/stanford/nlp/sempre/Grammar.java b/src/edu/stanford/nlp/sempre/Grammar.java index d04e6c7..38d953b 100644 --- a/src/edu/stanford/nlp/sempre/Grammar.java +++ b/src/edu/stanford/nlp/sempre/Grammar.java @@ -13,16 +13,25 @@ import java.util.*; /** * The grammar is a set of rules of the form: * (rule lhsCat (rhs ... rhs) semanticFn (key value) ... (key value)) + + * Some programming-language-esque features are supported to make life easier. + * All grammar-file variables should start with '@' (both loop variables and global definitions). + * Includes (reads in commands from the given file): + * (include base.grammar) + * Control flow: + * (when (and tag1 (not tag2)) ...) + * Macro definitions: + * (def @type fb:type.object.type) + * Iteration: + * (for @x (a an the) (rule $DT (@x) (IdentityFn))) * * @author Percy Liang */ public class Grammar { public static class Options { - @Option public List inPaths = new ArrayList(); + @Option public List inPaths = new ArrayList<>(); @Option(gloss = "Variables which are used to interpret the grammar file") - public List tags = new ArrayList(); - @Option - public String semanticFnPackage = "edu.stanford.nlp.sempre"; // TODO (rf) Move to SemanticFn.opts + public List tags = new ArrayList<>(); @Option public boolean binarizeRules = true; } @@ -30,12 +39,14 @@ public class Grammar { // All the rules in the grammar. Each parser can read these and transform // them however the parser wishes. - ArrayList rules = new ArrayList(); + // This contains binarized rules + ArrayList rules = new ArrayList<>(); + public List getRules() { return rules; } - Map macros = new HashMap(); // Map from macro name to its replacement value + Map macros = new HashMap<>(); // Map from macro name to its replacement value - // Verbatim copy of all the lines read, so we can output everything back. - List statements = new ArrayList(); + // Verbatim copy of all the lines read, so we can preserve the grammar file. + List statements = new ArrayList<>(); public void read() { LogInfo.begin_track("Grammar.read"); @@ -53,7 +64,7 @@ public class Grammar { private void verifyValid() { // Make sure that all the categories which are used are actually defined. - Set defined = new HashSet(); + Set defined = new HashSet<>(); defined.add(Rule.tokenCat); defined.add(Rule.phraseCat); defined.add(Rule.lemmaTokenCat); @@ -65,7 +76,7 @@ public class Grammar { for (Rule rule : rules) { for (String item : rule.rhs) { if (Rule.isCat(item) && !defined.contains(item)) { - throw new RuntimeException("Category not defined in the grammar: " + item + "; used in rule: " + rule); + LogInfo.warnings("Category not defined in the grammar: %s; used in rule: %s", item, rule); } } } @@ -90,7 +101,7 @@ public class Grammar { } // Replace all leaves of LispTree with value in macros if exists - public static LispTree applyMacros(Map macros, LispTree tree) { + private static LispTree applyMacros(Map macros, LispTree tree) { if (tree.isLeaf()) { LispTree replacement = macros.get(tree.value); if (replacement != null) return replacement; @@ -102,7 +113,7 @@ public class Grammar { return newTree; } - // Use macros associated with the grammar + // Apply the macro substitutions to |tree|. public LispTree applyMacros(LispTree tree) { return applyMacros(this.macros, tree); } @@ -141,8 +152,6 @@ public class Grammar { String command = tree.child(0).value; if ("rule".equals(command)) { interpretRule(tree); - } else if ("rhs_rules".equals(command)) { - interpretRhsRules(tree); } else if ("include".equals(command)) { if (path == null) { throw new RuntimeException( @@ -157,6 +166,8 @@ public class Grammar { } } else if ("def".equals(command)) { interpretMacroDef(tree); + } else if ("for".equals(command)) { + interpretFor(path, tree, tags); } else { throw new RuntimeException("Invalid command: " + command); } @@ -176,11 +187,31 @@ public class Grammar { throw new RuntimeException("Expected a single tag, but got: " + tree); } - private void interpretMacroDef(LispTree tree) { + public void interpretMacroDef(LispTree tree) { if (tree.children.size() != 3 || !tree.child(1).isLeaf()) { - throw new RuntimeException("Invalid usage: (def |name| |value|)"); + throw new RuntimeException("Invalid usage: (def |name| |value|)"); } - macros.put(tree.child(1).toString(), tree.child(2)); + String var = tree.child(1).value; + checkIsValidVar(var); + macros.put(var, applyMacros(tree.child(2))); + } + + public void interpretFor(String path, LispTree tree, Set tags) { + if (tree.children.size() <= 3 || !tree.child(1).isLeaf()) { + throw new RuntimeException("Invalid usage: (for |var| (|value| ... |value|) |statement| ...)"); + } + + String var = tree.child(1).value; + checkIsValidVar(var); + List values = applyMacros(tree.child(2)).children; + LispTree old = macros.get(var); + for (LispTree value : values) { + macros.put(var, value); + for (int j = 3; j < tree.children.size(); j++) + interpret(path, tree.child(j), tags); + } + if (old == null) macros.remove(var); + else macros.put(var, old); } private void interpretRule(LispTree tree) { @@ -188,6 +219,7 @@ public class Grammar { throw new RuntimeException("Invalid rule: " + tree); // (rule lhs rhs semantics (key value) ... (key value)) + tree = applyMacros(tree); // Parse LHS if (!tree.child(1).isLeaf()) @@ -196,7 +228,7 @@ public class Grammar { // Parse RHS List rhs = Lists.newArrayList(); - List isOptionals = new ArrayList(); + List isOptionals = new ArrayList<>(); LispTree rhsTree = tree.child(2); if (rhsTree.isLeaf()) throw new RuntimeException("RHS needs to be list, but got: " + rhsTree); @@ -234,28 +266,19 @@ public class Grammar { } } - rules.addAll(binarizeRule(rule, isOptionals)); + addRule(rule, isOptionals); } - // Allow multiple values in rhs to map to the same ConstantFn - // Example: (rhs_rules $A (largest biggest) (IdentityFn)) - private void interpretRhsRules(LispTree tree) { - // FIXME Doesn't handle optional - LispTree rhs = tree.child(2); - // TODO Way to just copy tree and substitute rhs? - for (LispTree rhsChild : rhs.children) { - LispTree newTree = LispTree.proto.newList(); - for (int i = 0; i < tree.children.size(); i++) { - if (i == 2) { - LispTree newRhs = LispTree.proto.newList(); - newRhs.addChild(rhsChild); - newTree.addChild(newRhs); - } - else - newTree.addChild(tree.child(i)); - } - interpretRule(newTree); - } + public synchronized boolean addRule(Rule rule) { + List isOptionals = new ArrayList<>(); + for (String rhs : rule.rhs) isOptionals.add(false); + return addRule(rule, isOptionals); + } + + // Add a rule to the grammar. + public synchronized boolean addRule(Rule rule, List isOptionals) { + rules.addAll(binarizeRule(rule, isOptionals)); + return true; } // Generate intermediate categories for binarization. @@ -263,14 +286,53 @@ public class Grammar { private String generateFreshCat() { freshCatIndex++; return "$Intermediate" + freshCatIndex; + // return "$I" + freshCatIndex + ":" + content; } - // Create multiple copies of this rule if there are optional RHS. + // Create multiple versions of this rule if there are optional RHS. // Restriction: must be able to split the RHS into two halves, each of // which contains at most one non-optional category. + // Recall that only the non-optional categories on the RHS are arguments into the SemanticFn. // Example: stop? $A stop $Stop? $B stop $Stop? private List binarizeRule(Rule rule, List isOptionals) { - List newRules = new ArrayList(); + List newRules = new ArrayList<>(); + + // Special case: JoinFn with an arg0Fn but with multiple non-optional categories. + // In this case, we need to use arg0Fn on just the first such category, and + // then use function application on the rest. + // Old: (rule $A (a ($Z optional) $B $C $D) (JoinFn (arg0 (lambda b (lambda c (lambda d ...)))))) + // New rules to binarize ($B is the first non-optional category): + // 1. (rule $I1 (a ($Z optional) $B) (JoinFn (arg0 (lambda b (lambda c (lambda d ...)))))) + // 2. (rule $A ($I1 $C $D) (JoinFn forward betaReduce)) + if (rule.sem instanceof JoinFn && ((JoinFn) rule.sem).getArg0Fn() != null) { + // Find the first non-optional category + int i = 0; + while (i < rule.rhs.size() && !(Rule.isCat(rule.rhs.get(i)) && !isOptionals.get(i))) + i++; + // Find the next non-optional category + int j = i + 1; + while (j < rule.rhs.size() && !(Rule.isCat(rule.rhs.get(i)) && !isOptionals.get(i))) + j++; + // If one exists, then we have to invoke special binarization + if (j < rule.rhs.size()) { + // Create an intermediate category + String intCat = generateFreshCat(); + + // Add rule 1 + List rhs1 = new ArrayList<>(rule.rhs.subList(0, i + 1)); + newRules.addAll(binarizeRule(new Rule(intCat, rhs1, rule.sem).setInfo(rule), isOptionals.subList(0, i + 1))); + + // Add rule 2 + List rhs2 = new ArrayList<>(); + rhs2.add(intCat); + rhs2.addAll(rule.rhs.subList(i + 1, rule.rhs.size())); + SemanticFn forwardBetaReduce = new JoinFn(); + forwardBetaReduce.init(LispTree.proto.parseFromString("(JoinFn forward betaReduce)")); + newRules.addAll(binarizeRule(new Rule(rule.lhs, rhs2, forwardBetaReduce).setInfo(rule), isOptionals.subList(i, isOptionals.size()))); + + return newRules; + } + } // Don't binarize: do same as before if (!opts.binarizeRules) { @@ -290,9 +352,9 @@ public class Grammar { } // Stores the current RHS that we're building up. - List newRhs = new ArrayList(); - List newIsOptional = new ArrayList(); - List isRequiredCat = new ArrayList(); // These are the arguments to the semantic function + List newRhs = new ArrayList<>(); + List newIsOptional = new ArrayList<>(); + List isRequiredCat = new ArrayList<>(); // These are the arguments to the semantic function // Left-binarize. assert rule.rhs.size() >= 2; @@ -305,9 +367,7 @@ public class Grammar { // Aim is to create rules with two RHS required categories // (binarized rules + tokens which don't cost anything). // Sometimes semantic functions take more than one argument. - // Important: we assume they just left-binarize! - // Works for things like ConcatFn, but not really for JoinFn (which if - // anything should be right-binarized). + // Note: we assume they just left-binarize. if (newRhs.size() < 2) // This should only happen in the beginning continue; @@ -315,6 +375,8 @@ public class Grammar { appliedRuleSem = true; boolean atEnd = (i == rule.rhs.size() - 1); + // TODO(pliang): currently, too many intermediate categories are created. Remove + // to make flatter grammars (will generate fewer derivations). String lhs = atEnd && appliedRuleSem ? rule.lhs : generateFreshCat(); // Create rule with newRhs possibly excluding the optionals (there should be at most 2) @@ -343,7 +405,7 @@ public class Grammar { // We can't allow empty RHS, but if we need it, just mark it as all // can be optional. if (rhs.size() > 0) - newRules.add(new Rule(lhs, rhs, sem)); + newRules.add(new Rule(lhs, rhs, sem).setInfo(rule)); else allCanBeOptional = true; } @@ -361,27 +423,55 @@ public class Grammar { // Final unary rule if needed if (!appliedRuleSem) - newRules.add(new Rule(rule.lhs, Lists.newArrayList(newRhs), rule.sem)); + newRules.add(new Rule(rule.lhs, Lists.newArrayList(newRhs), rule.sem).setInfo(rule)); - //LogInfo.begin_track("binarize %s", rule); - //for (Rule r : newRules) LogInfo.logs("%s", r); - //LogInfo.end_track(); + // LogInfo.begin_track("binarize %s", rule); + // for (Rule r : newRules) LogInfo.logs("%s", r); + // LogInfo.end_track(); return newRules; } + // Examples of |tree| + // (ConstantFn null) + // (NumberFn) private SemanticFn parseSemanticFn(LispTree tree) { - String name = tree.child(0).value; - if (name.equals("ConstantFn")) { - tree = applyMacros(tree); + // Syntactic sugar: foo => (ConstantFn foo) + if (tree.isLeaf()) { + LispTree newTree = LispTree.proto.newList(); + newTree.addChild("ConstantFn"); + newTree.addChild(tree.value); + tree = newTree; } - SemanticFn fn = null; - fn = (SemanticFn) Utils.newInstanceHard(opts.semanticFnPackage + "." + name); + String name = tree.child(0).value; + + // Syntactic sugar: (lambda x (var x)) => (JoinFn betaReduce forward (arg0 (lambda x (var x)))) + if (name.equals("lambda")) { + LispTree newTree = LispTree.proto.newList(); + newTree.addChild("JoinFn"); + newTree.addChild("betaReduce"); + newTree.addChild("forward"); + newTree.addChild(LispTree.proto.newList("arg0", tree)); + tree = newTree; + name = tree.child(0).value; + } + + // For backward compatibility: SemanticFn which have moved. + if (name.equals("LexiconFn") || name.equals("BridgeFn")) + name = "freebase." + name; + + SemanticFn fn; + fn = (SemanticFn) Utils.newInstanceHard(SempreUtils.resolveClassName(name)); if (fn == null) throw new RuntimeException("Invalid SemanticFn name: " + name); fn.init(tree); return fn; } + + private void checkIsValidVar(String var) { + if (!var.startsWith("@")) + LogInfo.errors("Invalid variable: '%s' doesn't start with '@'", var); + } } diff --git a/src/edu/stanford/nlp/sempre/HasScore.java b/src/edu/stanford/nlp/sempre/HasScore.java new file mode 100644 index 0000000..8fd1b49 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/HasScore.java @@ -0,0 +1,8 @@ +package edu.stanford.nlp.sempre; + +/** + * Things that have a score that is a dot product of weights and features + */ +public interface HasScore { + double getScore(); +} diff --git a/src/edu/stanford/nlp/sempre/IdentityFn.java b/src/edu/stanford/nlp/sempre/IdentityFn.java index 1ec15c3..487efc7 100644 --- a/src/edu/stanford/nlp/sempre/IdentityFn.java +++ b/src/edu/stanford/nlp/sempre/IdentityFn.java @@ -1,27 +1,20 @@ package edu.stanford.nlp.sempre; -import java.util.Collections; -import java.util.List; - /** * Identity function. * * @author Percy Liang */ public class IdentityFn extends SemanticFn { - @Override - public boolean equals(Object o) { - if (this == o) return true; - return o != null && getClass() == o.getClass(); - } - - public List call(Example ex, Callable c) { - if (c.getChildren().size() != 1) - throw new RuntimeException("Bad args: " + c.getChildren()); - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .withFormulaFrom(c.child(0)) - .createDerivation()); + public DerivationStream call(Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + return new Derivation.Builder() + .withCallable(c) + .withFormulaFrom(c.child(0)) + .createDerivation(); + } + }; } } diff --git a/src/edu/stanford/nlp/sempre/JavaExecutor.java b/src/edu/stanford/nlp/sempre/JavaExecutor.java index c7e3a83..4a84e56 100644 --- a/src/edu/stanford/nlp/sempre/JavaExecutor.java +++ b/src/edu/stanford/nlp/sempre/JavaExecutor.java @@ -8,6 +8,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -18,6 +19,8 @@ import java.util.Map; * @author Percy Liang */ public class JavaExecutor extends Executor { + private static JavaExecutor defaultExecutor = new JavaExecutor(); + // To simplify logical forms, define some shortcuts. private Map shortcuts = Maps.newHashMap(); @@ -28,6 +31,8 @@ public class JavaExecutor extends Executor { shortcuts.put("-", className + ".minus"); shortcuts.put("*", className + ".times"); shortcuts.put("/", className + ".divide"); + shortcuts.put("%", className + ".mod"); + shortcuts.put("!", className + ".not"); shortcuts.put("<", className + ".lessThan"); shortcuts.put("<=", className + ".lessThanEq"); @@ -36,13 +41,23 @@ public class JavaExecutor extends Executor { shortcuts.put(">=", className + ".greaterThanEq"); shortcuts.put("if", className + ".ifThenElse"); + shortcuts.put("map", className + ".map"); + shortcuts.put("reduce", className + ".reduce"); + shortcuts.put("select", className + ".select"); + shortcuts.put("range", className + ".range"); } - private static class BasicFunctions { + public static class BasicFunctions { public static double plus(double x, double y) { return x + y; } + public static int plus(int x, int y) { return x + y; } + public static int minus(int x, int y) { return x - y; } public static double minus(double x, double y) { return x - y; } + public static int times(int x, int y) { return x * y; } public static double times(double x, double y) { return x * y; } + public static int divide(int x, int y) { return x / y; } public static double divide(double x, double y) { return x / y; } + public static int mod(int x, int y) { return x % y; } + public static boolean not(boolean x) { return !x; } public static boolean lessThan(double x, double y) { return x < y; } public static boolean lessThanEq(double x, double y) { return x <= y; } @@ -51,67 +66,156 @@ public class JavaExecutor extends Executor { public static boolean greaterThanEq(double x, double y) { return x >= y; } public static Object ifThenElse(boolean b, Object x, Object y) { return b ? x : y; } + + // For very simple string concatenation + public static String plus(String a, String b) { return a + b; } + public static String plus(String a, String b, String c) { + return a + b + c; + } + public static String plus(String a, String b, String c, String d) { + return a + b + c + d; + } + public static String plus(String a, String b, String c, String d, String e) { + return a + b + c + d + e; + } + private static String toString(Object x) { + if (x instanceof String) + return (String) x; + else if (x instanceof Value) + return (x instanceof NameValue) ? ((NameValue) x).id : ((StringValue) x).value; + else + return null; + } + + // Apply func to each element of |list| and return the resulting list. + public static List map(List list, LambdaFormula func) { + List newList = new ArrayList(); + for (Object elem : list) { + Object newElem = apply(func, elem); + newList.add(newElem); + } + return newList; + } + + // list = [3, 5, 2], func = (lambda x (lambda y (call + (var x) (var y)))) + // Returns (3 + 5) + 2 = 10 + public static Object reduce(List list, LambdaFormula func) { + if (list.size() == 0) return null; + Object x = list.get(0); + for (int i = 1; i < list.size(); i++) + x = apply(func, x, list.get(i)); + return x; + } + + // Return elements x of |list| such that func(x) is true. + public static List select(List list, LambdaFormula func) { + List newList = new ArrayList(); + for (Object elem : list) { + Object test = apply(func, elem); + if ((Boolean) test) + newList.add(elem); + } + return newList; + } + + private static Object apply(LambdaFormula func, Object x) { + // Apply the function func to x. In order to do that, need to convert x into a value. + Formula formula = Formulas.lambdaApply(func, new ValueFormula(toValue(x))); + return defaultExecutor.processFormula(formula); + } + private static Object apply(LambdaFormula func, Object x, Object y) { + // Apply the function func to x and y. In order to do that, need to convert x into a value. + Formula formula = Formulas.lambdaApply(func, new ValueFormula(toValue(x))); + formula = Formulas.lambdaApply((LambdaFormula) formula, new ValueFormula(toValue(y))); + return defaultExecutor.processFormula(formula); + } + + public static List range(int start, int end) { + List result = new ArrayList(); + for (int i = start; i < end; i++) + result.add(i); + return result; + } } - public Response execute(Formula formula) { + public Response execute(Formula formula, ContextValue context) { + // We can do beta reduction here since macro substitution preserves the + // denotation (unlike for lambda DCS). formula = Formulas.betaReduction(formula); - return new Response(recurse(formula)); + + try { + return new Response(toValue(processFormula(formula))); + } catch (Exception e) { + e.printStackTrace(); + return new Response(ErrorValue.badJava(e.toString())); + } } - private Value recurse(Formula formula) { - if (formula instanceof ValueFormula) - return ((ValueFormula)formula).value; + private Object processFormula(Formula formula) { + if (formula instanceof ValueFormula) // Unpack value and convert to object (e.g., for ints) + return toObject(((ValueFormula) formula).value); - if (formula instanceof CallFormula) { + if (formula instanceof CallFormula) { // Invoke the function. // Recurse - CallFormula call = (CallFormula)formula; - Value func = recurse(call.func); - List args = Lists.newArrayList(); - for (Formula arg : call.args) - args.add(recurse(arg)); + CallFormula call = (CallFormula) formula; + Object func = processFormula(call.func); + List args = Lists.newArrayList(); + for (Formula arg : call.args) { + args.add(processFormula(arg)); + } if (!(func instanceof NameValue)) throw new RuntimeException("Invalid func: " + call.func + " => " + func); - String id = ((NameValue)func).id; + String id = ((NameValue) func).id; id = MapUtils.get(shortcuts, id, id); if (id.startsWith(".")) // Instance method - return toValue(invoke(id.substring(1), toObject(args.get(0)), toObjects(args.subList(1, args.size())))); - else - return toValue(invoke(id, null, toObjects(args))); + return invoke(id.substring(1), args.get(0), args.subList(1, args.size()).toArray(new Object[0])); + else // Static method + return invoke(id, null, args.toArray(new Object[0])); } - throw new RuntimeException("Invalid formula: " + formula); + // Just pass it through... + return formula; } - private Value toValue(Object obj) { - if (obj instanceof Boolean) return new BooleanValue((Boolean)obj); - if (obj instanceof Integer) return new NumberValue(((Integer)obj).intValue()); - if (obj instanceof Double) return new NumberValue(((Double)obj).doubleValue()); - if (obj instanceof String) return new StringValue((String)obj); - throw new RuntimeException("Invalid object: " + obj); + // Convert the Object back to a Value + private static Value toValue(Object obj) { + if (obj instanceof Value) return (Value) obj; + if (obj instanceof Boolean) return new BooleanValue((Boolean) obj); + if (obj instanceof Integer) return new NumberValue(((Integer) obj).intValue()); + if (obj instanceof Double) return new NumberValue(((Double) obj).doubleValue()); + if (obj instanceof String) return new StringValue((String) obj); + if (obj instanceof List) { + List list = Lists.newArrayList(); + for (Object elem : (List) obj) + list.add(toValue(elem)); + return new ListValue(list); + } + throw new RuntimeException("Unhandled object: " + obj + " with class " + obj.getClass()); } - private Object[] toObjects(List values) { - Object[] result = new Object[values.size()]; - for (int i = 0; i < values.size(); i++) - result[i] = toObject(values.get(i)); - return result; - } - - private Object toObject(Value value) { + // Convert a Value (which are specified in the formulas) to an Object (which + // many Java functions take). + private static Object toObject(Value value) { if (value instanceof NumberValue) { - // Unfortunately, we don't make a distinction between ints and doubles, so this is a hack. - double x = ((NumberValue)value).value; - if (x == (int)x) return new Integer((int)x); - return new Double((int)x); + // Unfortunately, NumberValues don't make a distinction between ints and + // doubles, so this is a hack. + double x = ((NumberValue) value).value; + if (x == (int) x) return new Integer((int) x); + return new Double(x); } else if (value instanceof BooleanValue) { - return ((BooleanValue)value).value; + return ((BooleanValue) value).value; } else if (value instanceof StringValue) { - return ((StringValue)value).value; + return ((StringValue) value).value; + } else if (value instanceof ListValue) { + List list = Lists.newArrayList(); + for (Value elem : ((ListValue) value).values) + list.add(toObject(elem)); + return list; } else { - throw new RuntimeException("Unhandled: " + value); + return value; // Preserve the Value (which can be an object) } } @@ -124,9 +228,11 @@ public class JavaExecutor extends Executor { if (isStatic) { // Static methods int i = id.lastIndexOf('.'); - if (i == -1) throw new RuntimeException("Expected ., but got: " + id); + if (i == -1) { + throw new RuntimeException("Expected ., but got: " + id); + } String className = id.substring(0, i); - methodName = id.substring(i+1); + methodName = id.substring(i + 1); try { cls = Class.forName(className); @@ -142,41 +248,82 @@ public class JavaExecutor extends Executor { // Find a suitable method List nameMatches = Lists.newArrayList(); + Method bestMethod = null; + int bestCost = INVALID_TYPE_COST; for (Method m : methods) { if (!m.getName().equals(methodName)) continue; nameMatches.add(m); if (isStatic != Modifier.isStatic(m.getModifiers())) continue; - if (!isCompatible(m.getParameterTypes(), args)) continue; - //logs(" %s", m); + int cost = typeCastCost(m.getParameterTypes(), args); + if (cost < bestCost) { + bestCost = cost; + bestMethod = m; + } + } + + if (bestMethod != null) { try { - return m.invoke(thisObj, args); + return bestMethod.invoke(thisObj, args); } catch (InvocationTargetException e) { throw new RuntimeException(e.getCause()); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } - throw new RuntimeException("Method " + methodName + " not found in class " + cls + " with arguments " + Arrays.asList(args) + "; candidates: " + nameMatches); + List types = Lists.newArrayList(); + for (Object arg : args) + types.add(arg.getClass().toString()); + throw new RuntimeException("Method " + methodName + " not found in class " + cls + " with arguments " + Arrays.asList(args) + " having types " + types + "; candidates: " + nameMatches); } - private boolean isCompatible(Class[] types, Object[] args) { - if (types.length != args.length) return false; + private int typeCastCost(Class[] types, Object[] args) { + if (types.length != args.length) return INVALID_TYPE_COST; + int cost = 0; for (int i = 0; i < types.length; i++) { - if (!isCompatible(types[i], args[i])) - return false; + cost += typeCastCost(types[i], args[i]); + if (cost >= INVALID_TYPE_COST) { + // LogInfo.dbgs("NOT COMPATIBLE: want %s, got %s with type %s", types[i], args[i], args[i].getClass()); + break; + } } - return true; + return cost; } - private boolean isCompatible(Class type, Object arg) { - if (arg == null) return !type.isPrimitive(); - if (type.isInstance(arg)) return true; - // Order matters here for type checking: try to match the most restrictive types first. - if (type == Boolean.TYPE) return arg instanceof Boolean; - if (type == Integer.TYPE) return arg instanceof Integer || arg instanceof Long; - if (type == Long.TYPE) return arg instanceof Integer || arg instanceof Long; - if (type == Float.TYPE) return arg instanceof Integer || arg instanceof Long || arg instanceof Float || arg instanceof Double; - if (type == Double.TYPE) return arg instanceof Integer || arg instanceof Long || arg instanceof Float || arg instanceof Double; - return false; + // Return whether the object |arg| is compatible with |type|. + // 0: perfect match + // 1: don't match, but don't lose anything + // 2: don't match, and can lose something + // INVALID_TYPE_COST: impossible + private int typeCastCost(Class type, Object arg) { + if (arg == null) return !type.isPrimitive() ? 0 : INVALID_TYPE_COST; + if (type.isInstance(arg)) return 0; + if (type == Boolean.TYPE) return arg instanceof Boolean ? 0 : INVALID_TYPE_COST; + else if (type == Integer.TYPE) { + if (arg instanceof Integer) return 0; + if (arg instanceof Long) return 1; + return INVALID_TYPE_COST; + } + if (type == Long.TYPE) { + if (arg instanceof Integer) return 1; + if (arg instanceof Long) return 0; + return INVALID_TYPE_COST; + } + if (type == Float.TYPE) { + if (arg instanceof Integer) return 1; + if (arg instanceof Long) return 1; + if (arg instanceof Float) return 0; + if (arg instanceof Double) return 2; + return INVALID_TYPE_COST; + } + if (type == Double.TYPE) { + if (arg instanceof Integer) return 1; + if (arg instanceof Long) return 1; + if (arg instanceof Float) return 1; + if (arg instanceof Double) return 0; + return INVALID_TYPE_COST; + } + return INVALID_TYPE_COST; } + + private static final int INVALID_TYPE_COST = 1000; } diff --git a/src/edu/stanford/nlp/sempre/JoinFn.java b/src/edu/stanford/nlp/sempre/JoinFn.java index d7ed7e3..b5122fd 100644 --- a/src/edu/stanford/nlp/sempre/JoinFn.java +++ b/src/edu/stanford/nlp/sempre/JoinFn.java @@ -4,6 +4,7 @@ import fig.basic.LispTree; import fig.basic.LogInfo; import fig.basic.Option; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -20,9 +21,10 @@ import java.util.List; */ public class JoinFn extends SemanticFn { public static class Options { - @Option(gloss = "whether to do a hard type-check") - public boolean hardTypeCheck = true; @Option(gloss = "Verbose") public int verbose = 0; + @Option public boolean showTypeCheckFailures = false; + @Option public boolean typeInference = false; + @Option public boolean specializedTypeCheck = true; } public static Options opts = new Options(); @@ -38,7 +40,7 @@ public class JoinFn extends SemanticFn { // 2. arg0 = fb:en.barack_obama, binary = (lambda x (fb:people.person.place_of_birth (var x))), arg1 = fb:en.honolulu // From a function application viewpoint, arg1 is the argument and arg0 is the return type. // The unary can be placed into arg0 of the binary or arg1 of the binary. - // When we write a join (binary unary), unary goes into arg1, so if we want the unary to go into arg1, we need to reverse the binary. + // When we write a join (binary unary), unary goes into arg1, so if we want the unary to go into arg0, we need to reverse the binary. private boolean unaryCanBeArg0 = false; private boolean unaryCanBeArg1 = false; @@ -53,33 +55,44 @@ public class JoinFn extends SemanticFn { * - has binary[parents] unary[Obama] (binary,unary unaryCanBeArg1) */ - // Optionally specify the first of the two arguments, + // Optionally specify the first of the two arguments to the JoinFn, // in which case, this function should only be called on one argument. + // Note: this is confusing - arg0 here refers to the arguments to JoinFn, not + // the arg0 and arg1 of the binary. private ConstantFn arg0Fn = null; + public ConstantFn getArg0Fn() { return arg0Fn; } public void init(LispTree tree) { super.init(tree); for (int j = 1; j < tree.children.size(); j++) { String arg = tree.child(j).value; if (tree.child(j).isLeaf()) { - if ("binary,unary".equals(arg)) - unaryFirst = false; - else if ("unary,binary".equals(arg)) - unaryFirst = true; - else if ("unaryCanBeArg0".equals(arg)) - unaryCanBeArg0 = true; - else if ("unaryCanBeArg1".equals(arg)) - unaryCanBeArg1 = true; - else if ("forward".equals(arg)) { - unaryFirst = false; - unaryCanBeArg1 = true; - } else if ("backward".equals(arg)) { - unaryFirst = true; - unaryCanBeArg1 = true; - } else if ("betaReduce".equals(arg)) { - betaReduce = true; - } else { - throw new RuntimeException("Invalid argument: " + arg); + switch (arg) { + case "binary,unary": + unaryFirst = false; + break; + case "unary,binary": + unaryFirst = true; + break; + case "unaryCanBeArg0": + unaryCanBeArg0 = true; + break; + case "unaryCanBeArg1": + unaryCanBeArg1 = true; + break; + case "forward": + unaryFirst = false; + unaryCanBeArg1 = true; + break; + case "backward": + unaryFirst = true; + unaryCanBeArg1 = true; + break; + case "betaReduce": + betaReduce = true; + break; + default: + throw new RuntimeException("Invalid argument: " + arg); } } else { if ("arg0".equals(tree.child(j).child(0).value)) { @@ -95,133 +108,164 @@ public class JoinFn extends SemanticFn { throw new RuntimeException("At least one of unaryCanBeArg0 and unaryCanBeArg1 must be set"); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - JoinFn joinFn = (JoinFn) o; - if (unaryCanBeArg0 != joinFn.unaryCanBeArg0) return false; - if (unaryCanBeArg1 != joinFn.unaryCanBeArg1) return false; - if (unaryFirst != joinFn.unaryFirst) return false; - if (betaReduce != joinFn.betaReduce) return false; - if (arg0Fn != null ? !arg0Fn.equals(joinFn.arg0Fn) : joinFn.arg0Fn != null) - return false; - return true; + public DerivationStream call(Example ex, Callable c) { + return new LazyJoinFnDerivs(ex, c); } - private List intersect(List types0, List types1) { - List types = new ArrayList(); - for (String t : types0) - if (types1.contains(t)) - types.add(t); - return types; - } + public class LazyJoinFnDerivs extends MultipleDerivationStream { + private int currIndex = 0; + private List derivations = new ArrayList<>(); + private Example ex; + private Callable callable; + Derivation unaryDeriv, binaryDeriv; - public List call(Example ex, Callable c) { - Derivation child0 = null; - Derivation child1 = null; + public LazyJoinFnDerivs(Example ex, Callable c) { + this.ex = ex; + this.callable = c; + Derivation child0, child1; + // TODO(pliang): we can actually push most of this logic into createDerivation() + // don't need to get the exact size - if (arg0Fn != null) { - if (c.getChildren().size() != 1) - throw new RuntimeException( - "Expected one argument (already have " + arg0Fn + - "), but got args: " + c.getChildren()); - // This is just a virtual child which is not a derivation. - child0 = arg0Fn.call(ex, CallInfo.NULL_INFO).get(0); - child1 = c.child(0); - } else { - if (c.getChildren().size() != 2) - throw new RuntimeException("Expected two arguments, but got: " + c.getChildren()); - child0 = c.child(0); - child1 = c.child(1); - } + if (arg0Fn != null) { + if (c.getChildren().size() != 1) + throw new RuntimeException("Expected one argument (already have " + arg0Fn + "), but got args: " + c.getChildren()); + // This is just a virtual child which is not a derivation. + DerivationStream ld = arg0Fn.call(ex, CallInfo.NULL_INFO); + child0 = ld.next(); + child1 = c.child(0); + } else { + if (c.getChildren().size() != 2) + throw new RuntimeException("Expected two arguments, but got: " + c.getChildren()); + child0 = c.child(0); + child1 = c.child(1); + } - Derivation binaryDeriv, unaryDeriv; - if (unaryFirst) { - unaryDeriv = child0; - binaryDeriv = child1; - } else { - binaryDeriv = child0; - unaryDeriv = child1; - } - - List results = new ArrayList(); - performJoins(ex, c, binaryDeriv, unaryDeriv, results); - return results; - } - - private void performJoins(Example ex, Callable c, Derivation binaryDeriv, Derivation unaryDeriv, List results) { - // Note that the derivations might not start or end because they're not anchored in the text. - String binaryPos = binaryDeriv.start == -1 ? "NONE" : ex.languageInfo.getCanonicalPos(binaryDeriv.start); - String unaryPos = unaryDeriv.start == -1 ? "NONE" : ex.languageInfo.getCanonicalPos(unaryDeriv.start); - if (unaryCanBeArg1) { - results.addAll( - performJoin( - ex, c, - binaryDeriv, binaryDeriv.formula, binaryDeriv.type, - unaryDeriv, unaryDeriv.formula, unaryDeriv.type, - "binary=" + binaryPos + ",unary=" + unaryPos)); - } - if (unaryCanBeArg0) { - results.addAll( - performJoin( - ex, c, - binaryDeriv, FbFormulasInfo.reverseFormula(binaryDeriv.formula), binaryDeriv.type.reverse(), - unaryDeriv, unaryDeriv.formula, unaryDeriv.type, - "binary=" + binaryPos + ",unary=" + unaryPos + "_reverse")); - } - } - - public List performJoin(Example ex, Callable c, - Derivation binaryDeriv, Formula binaryFormula, SemType binaryType, - Derivation unaryDeriv, Formula unaryFormula, SemType unaryType, - String featureDesc) { - FeatureVector features = new FeatureVector(); - SemType type = binaryType.apply(unaryType); - - // Add features - if (FeatureExtractor.containsDomain("joinPos") && featureDesc != null) - features.add("joinPos", featureDesc); - - if (opts.verbose >= 3) { - LogInfo.logs( - "JoinFn: binary: %s [%s], unary: %s [%s], result: [%s]", - binaryFormula, binaryType, unaryFormula, unaryType, type); - } - - if (!type.isValid()) { - if (opts.hardTypeCheck) - return Collections.emptyList(); // Don't accept logical forms that don't type check - else { - if (FeatureExtractor.containsDomain("typeCheck")) - features.add("typeCheck", "joinMismatch"); // Just add a feature + if (unaryFirst) { + unaryDeriv = child0; + binaryDeriv = child1; + } else { + binaryDeriv = child0; + unaryDeriv = child1; } } - Formula f; - if (betaReduce) { - if (!(binaryFormula instanceof LambdaFormula)) - throw new RuntimeException("Expected LambdaFormula as the binary, but got: " + binaryFormula + ", unary is " + unaryFormula); - f = Formulas.lambdaApply((LambdaFormula)binaryFormula, unaryFormula); - } else { - f = new JoinFormula(binaryFormula, unaryFormula); - } - - FbFormulasInfo.touchBinaryFormula(binaryFormula); - Derivation newDeriv = new Derivation.Builder() - .withCallable(c) - .formula(f) - .type(type) - .localFeatureVector(features) - .createDerivation(); - - if (SemanticFn.opts.trackLocalChoices) { - newDeriv.localChoices.add( - "JoinFn " + - binaryDeriv.startEndString(ex.getTokens()) + " " + binaryDeriv.formula + " AND " + - unaryDeriv.startEndString(ex.getTokens()) + " " + unaryDeriv.formula); + @Override + public int estimatedSize() { + return 2; // This is an upper bound } - return Collections.singletonList(newDeriv); + @Override + public Derivation createDerivation() { + if (currIndex == 0) + doJoins(binaryDeriv, unaryDeriv); + if (currIndex == derivations.size()) + return null; + return derivations.get(currIndex++); + } + + SemType specializedTypeCheck(SemType binaryType, SemType unaryType) { + // Ugly special case for Free917/WebQuestions: when |that| is a + // UnionSemType corresponding to an entity (e.g., + // fb:en.the_washington_post) and we are joining with a relation (e.g., + // fb:cvg.game_version.publisher), then we end up calling this function + // with: + // - that: (union fb:business.employer ...) + // - argType: fb:cvg.cvg_publisher + // The meet here is fb:cvg.cvg_publisher, but we actually want to return bottom (to be more stringent). + SemType argType = binaryType.getArgType(); + if (unaryType instanceof TopSemType) // Happens when we don't know the type of the unary + return SemType.bottomType; + if (unaryType instanceof AtomicSemType) // Make things uniform + unaryType = new UnionSemType(unaryType); + if (unaryType instanceof UnionSemType && argType instanceof AtomicSemType) { + for (SemType t : ((UnionSemType) unaryType).baseTypes) + if (t instanceof AtomicSemType && + SemTypeHierarchy.singleton.getSupertypes(((AtomicSemType) t).name).contains(((AtomicSemType) argType).name)) + return binaryType.getRetType(); + return SemType.bottomType; + } + return binaryType.apply(unaryType); // Default + } + + // Return null if unable to join. + private Derivation doJoin(Derivation binaryDeriv, Formula binaryFormula, SemType binaryType, + Derivation unaryDeriv, Formula unaryFormula, SemType unaryType, + String featureDesc) { + // Do a coarse type check. + SemType type = opts.specializedTypeCheck ? specializedTypeCheck(binaryType, unaryType) : binaryType.apply(unaryType); + if (!type.isValid()) { + if (opts.showTypeCheckFailures) + LogInfo.warnings("JoinFn: type check failed: [%s : %s] JOIN [%s : %s]", + binaryFormula, binaryType, + unaryFormula, unaryType); + return null; + } + + Formula f; + if (betaReduce) { + if (!(binaryFormula instanceof LambdaFormula)) + throw new RuntimeException("Expected LambdaFormula as the binary, but got: " + binaryFormula + ", unary is " + unaryFormula); + f = Formulas.lambdaApply((LambdaFormula) binaryFormula, unaryFormula); + } else { + f = new JoinFormula(binaryFormula, unaryFormula); + } + + // Do full type inference. + if (opts.typeInference) { + SemType fullType = TypeInference.inferType(f); + if (opts.verbose >= 2) + LogInfo.logs("JoinFn.typeInference: %s => %s [coarse type = %s]", f, fullType, type); + if (!fullType.isValid()) return null; // Rule out logical form + type = fullType; // Use the more specific type + } + + if (opts.verbose >= 3) { + LogInfo.logs( + "JoinFn: binary: %s [%s], unary: %s [%s], result: %s [%s]", + binaryFormula, binaryType, unaryFormula, unaryType, f, type); + } + + // Add features + FeatureVector features = new FeatureVector(); + if (FeatureExtractor.containsDomain("joinPos") && featureDesc != null) + features.add("joinPos", featureDesc); + + // FbFormulasInfo.touchBinaryFormula(binaryFormula); + Derivation newDeriv = new Derivation.Builder() + .withCallable(callable) + .formula(f) + .type(type) + .localFeatureVector(features) + .createDerivation(); + + if (SemanticFn.opts.trackLocalChoices) { + newDeriv.addLocalChoice( + "JoinFn " + + (binaryDeriv.start == -1 ? "-" : binaryDeriv.startEndString(ex.getTokens())) + " " + binaryDeriv.formula + " AND " + + (unaryDeriv.start == -1 ? "-" : unaryDeriv.startEndString(ex.getTokens())) + " " + unaryDeriv.formula); + } + + return newDeriv; + } + + private void doJoins(Derivation binaryDeriv, Derivation unaryDeriv) { + String binaryPos = ex.languageInfo.getCanonicalPos(binaryDeriv.start); + String unaryPos = ex.languageInfo.getCanonicalPos(unaryDeriv.start); + if (unaryCanBeArg0) { + Derivation deriv = doJoin( + binaryDeriv, Formulas.reverseFormula(binaryDeriv.formula), binaryDeriv.type.reverse(), + unaryDeriv, unaryDeriv.formula, unaryDeriv.type, + "binary=" + binaryPos + ",unary=" + unaryPos + "_reverse"); + if (deriv != null) derivations.add(deriv); + } + if (unaryCanBeArg1) { + Derivation deriv = doJoin( + binaryDeriv, binaryDeriv.formula, binaryDeriv.type, + unaryDeriv, unaryDeriv.formula, unaryDeriv.type, + "binary=" + binaryPos + ",unary=" + unaryPos); + if (deriv != null) derivations.add(deriv); + } + Collections.sort(derivations, Derivation.derivScoreComparator); + } } } diff --git a/src/edu/stanford/nlp/sempre/JoinFormula.java b/src/edu/stanford/nlp/sempre/JoinFormula.java index b7fc941..b4be44e 100644 --- a/src/edu/stanford/nlp/sempre/JoinFormula.java +++ b/src/edu/stanford/nlp/sempre/JoinFormula.java @@ -3,6 +3,8 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * A join formula represents a database join and has the following form: * (relation child) @@ -39,6 +41,16 @@ public class JoinFormula extends Formula { return result == null ? new JoinFormula(relation.map(func), child.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) { + res.addAll(relation.mapToList(func, alwaysRecurse)); + res.addAll(child.mapToList(func, alwaysRecurse)); + } + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof JoinFormula)) return false; @@ -47,7 +59,7 @@ public class JoinFormula extends Formula { if (!this.child.equals(that.child)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + relation.hashCode(); diff --git a/src/edu/stanford/nlp/sempre/Json.java b/src/edu/stanford/nlp/sempre/Json.java index 9a1e4fe..e61dd32 100644 --- a/src/edu/stanford/nlp/sempre/Json.java +++ b/src/edu/stanford/nlp/sempre/Json.java @@ -21,7 +21,7 @@ import java.util.Map; * @author Roy Frostig */ public final class Json { - private Json() {} + private Json() { } private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); static { @@ -113,6 +113,13 @@ public final class Json { } } + public static String prettyWriteValueAsStringHard(Object o) { + try { + return getMapper().writerWithDefaultPrettyPrinter().writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } public static String writeValueAsStringHard(Object o) { return writeValueAsStringHard(o, Object.class); } @@ -135,6 +142,17 @@ public final class Json { } } + public static void prettyWriteValueHard(File f, Object o) { + try { + getMapper().writerWithDefaultPrettyPrinter().writeValue(f, o); + } catch (JsonMappingException e) { + e.printStackTrace(); + } catch (JsonGenerationException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } public static void writeValueHard(File f, Object o) { writeValueHard(f, o, Object.class); } diff --git a/src/edu/stanford/nlp/sempre/KnowledgeGraph.java b/src/edu/stanford/nlp/sempre/KnowledgeGraph.java new file mode 100644 index 0000000..3934d2f --- /dev/null +++ b/src/edu/stanford/nlp/sempre/KnowledgeGraph.java @@ -0,0 +1,107 @@ +package edu.stanford.nlp.sempre; + +import java.lang.reflect.InvocationTargetException; +import java.util.*; + +import fig.basic.*; + + +/** + * Represents a small knowledge graph (much smaller than Freebase). + * + * A KnowledgeGraph can be created from either + * - a list of triples, or + * - other data format (e.g., web tables in CSV format) + * + * @author ppasupat + */ +public abstract class KnowledgeGraph { + + public static KnowledgeGraph fromLispTree(LispTree tree) { + if ("graph".equals(tree.child(0).value)) { + if (tree.children.size() > 1 && tree.child(1).isLeaf()) { + // Use a specific subclass of KnowledgeGraph + try { + String className = tree.child(1).value; + Class classObject = Class.forName(SempreUtils.resolveClassName(className)); + return (KnowledgeGraph) classObject.getDeclaredMethod("fromLispTree", LispTree.class).invoke(null, tree); + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException + | NoSuchMethodException | SecurityException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } else { + // (graph (a1 r1 b1) (a2 r2 b2) ...) -- explicit triples + return NaiveKnowledgeGraph.fromLispTree(tree); + } + } else { + throw new RuntimeException("Cannot convert " + tree + " to KnowledgeGraph."); + } + } + + // ============================================================ + // Helper methods + // ============================================================ + + /** + * Return the reversed relation if |r| is of the form |!relation|. + * Otherwise, return null. + */ + public static Value isReversedRelation(Value r) { + if (r instanceof NameValue) { + String id = ((NameValue) r).id; + if (id.startsWith("!")) return new NameValue(id.substring(1)); + } + return null; + } + + /** Convert between |r| and |!r| (r must be a NameValue) */ + public static Value getReversedPredicate(Value r) { + if (r instanceof NameValue) { + String id = ((NameValue) r).id; + if (id.startsWith("!")) + return new NameValue(id.substring(1)); + else + return new NameValue("!" + id); + } else { + throw new BadFormulaException("Cannot reverse " + r + " which is not a NameValue"); + } + } + + /** Reverse the pairs */ + public static List> getReversedPairs(Collection> pairs) { + List> reversed = new ArrayList<>(); + for (Pair pair : pairs) + reversed.add(new Pair<>(pair.getSecond(), pair.getFirst())); + return reversed; + } + + // ============================================================ + // Abstract methods + // ============================================================ + + public abstract LispTree toLispTree(); + @Override public String toString() { return toLispTree().toString(); } + + /** Return all y such that x in firsts and (x,r,y) in graph */ + public abstract List joinFirst(Value r, Collection firsts); + + /** Return all x such that y in seconds and (x,r,y) in graph */ + public abstract List joinSecond(Value r, Collection seconds); + + /** Return all (x,y) such that x in firsts and (x,r,y) in graph */ + public abstract List> filterFirst(Value r, Collection firsts); + + /** Return all (x,y) such that y in seconds and (x,r,y) in graph */ + public abstract List> filterSecond(Value r, Collection seconds); + + // ============================================================ + // Fuzzy Matching + // ============================================================ + + /** Return all entities / unaries / binaries that approximately match the given term */ + public abstract Collection getFuzzyMatchedFormulas(String term, FuzzyMatchFn.FuzzyMatchFnMode mode); + + /** Return all entities / unaries / binaries */ + public abstract Collection getAllFormulas(FuzzyMatchFn.FuzzyMatchFnMode mode); + +} diff --git a/src/edu/stanford/nlp/sempre/LambdaFormula.java b/src/edu/stanford/nlp/sempre/LambdaFormula.java index fdf0694..57239a1 100644 --- a/src/edu/stanford/nlp/sempre/LambdaFormula.java +++ b/src/edu/stanford/nlp/sempre/LambdaFormula.java @@ -3,6 +3,8 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * Lambda abstraction (lambda |var| |body|) * Percy Liang @@ -29,13 +31,21 @@ public class LambdaFormula extends Formula { return result == null ? new LambdaFormula(var, body.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) + res.addAll(body.mapToList(func, alwaysRecurse)); + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof LambdaFormula)) return false; LambdaFormula that = (LambdaFormula) thatObj; return this.var.equals(that.var) && this.body.equals(that.body); } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + var.hashCode(); diff --git a/src/edu/stanford/nlp/sempre/LanguageAnalyzer.java b/src/edu/stanford/nlp/sempre/LanguageAnalyzer.java new file mode 100644 index 0000000..e2170a4 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/LanguageAnalyzer.java @@ -0,0 +1,31 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.*; + +/** + * LanguageAnalyzer takes an utterance and applies various NLP pre-processing steps to + * to output a LanguageInfo object + * + * @author Alex Ratner + */ +public abstract class LanguageAnalyzer { + public static class Options { + @Option public String languageAnalyzer = "SimpleAnalyzer"; + + @Option(gloss = "Whether to convert tokens in the utterance to lowercase") + public boolean lowerCaseTokens = true; + } + public static Options opts = new Options(); + + // We keep a singleton LanguageAnalyzer because for any given run we + // generally will be working with one. + private static LanguageAnalyzer singleton; + public static LanguageAnalyzer getSingleton() { + if (singleton == null) + singleton = (LanguageAnalyzer) Utils.newInstanceHard(SempreUtils.resolveClassName(opts.languageAnalyzer)); + return singleton; + } + public static void setSingleton(LanguageAnalyzer analyzer) { singleton = analyzer; } + + public abstract LanguageInfo analyze(String utterance); +} diff --git a/src/edu/stanford/nlp/sempre/LanguageInfo.java b/src/edu/stanford/nlp/sempre/LanguageInfo.java index 24fd832..c469c43 100644 --- a/src/edu/stanford/nlp/sempre/LanguageInfo.java +++ b/src/edu/stanford/nlp/sempre/LanguageInfo.java @@ -5,39 +5,20 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import edu.stanford.nlp.ling.CoreAnnotations; -import edu.stanford.nlp.ling.CoreAnnotations.*; -import edu.stanford.nlp.ling.CoreLabel; -import edu.stanford.nlp.pipeline.Annotation; -import edu.stanford.nlp.pipeline.StanfordCoreNLP; import fig.basic.IntPair; import fig.basic.LispTree; -import fig.basic.Option; import fig.basic.MemUsage; import java.util.*; /** - * Interface with Stanford CoreNLP to do basic things like POS tagging and NER. + * Represents an linguistic analysis of a sentence (provided by some LanguageAnalyzer). + * * @author akchou */ -@JsonIgnoreProperties(ignoreUnknown=true) +@JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class LanguageInfo implements MemUsage.Instrumented { - public static class Options { - @Option(gloss="What CoreNLP annotators to run") - public List annotators = Lists.newArrayList("tokenize", "ssplit", "pos", "lemma", "ner"); - - @Option(gloss="Whether to use CoreNLP annotators") - public boolean useAnnotators = true; - - @Option(gloss="Whether to be case sensitive") - public boolean caseSensitive = false; - } - - public static Options opts = new Options(); - public static StanfordCoreNLP pipeline = null; // Tokenization of input. @JsonProperty @@ -53,14 +34,37 @@ public class LanguageInfo implements MemUsage.Instrumented { @JsonProperty public final List nerValues; // NER values (contains times, dates, etc.) - private Map lemmaSpans; + private Map lemmaSpans; + + public static class DependencyEdge { + @JsonProperty + public final String label; // Dependency label + @JsonProperty + public final int modifier; // Position of modifier + + @JsonCreator + public DependencyEdge(@JsonProperty("label") String label, @JsonProperty("modifier") int modifier) { + this.label = label; + this.modifier = modifier; + } + + @Override + public String toString() { + return label + "->" + modifier; + } + } + + @JsonProperty + // Dependencies of each token, represented as a (relation, parentIndex) pair + public final List> dependencyChildren; public LanguageInfo() { this(new ArrayList(), new ArrayList(), new ArrayList(), new ArrayList(), - new ArrayList()); + new ArrayList(), + new ArrayList>()); } @JsonCreator @@ -68,50 +72,14 @@ public class LanguageInfo implements MemUsage.Instrumented { @JsonProperty("lemmaTokens") List lemmaTokens, @JsonProperty("posTags") List posTags, @JsonProperty("nerTags") List nerTags, - @JsonProperty("nerValues") List nerValues) { + @JsonProperty("nerValues") List nerValues, + @JsonProperty("dependencyChildren") List> dependencyChildren) { this.tokens = tokens; this.lemmaTokens = lemmaTokens; this.posTags = posTags; this.nerTags = nerTags; this.nerValues = nerValues; - } - - // TODO: don't muck with the POS tag; instead have a separate flag for isContent which looks at posTag != "MD" && lemma != "be" && lemma != "have" - // Need to update TextToTextMatcher - private static final String[] AUX_VERB_ARR = new String[]{"is", "are", "was", - "were", "am", "be", "been", "will", "shall", "have", "has", "had", - "would", "could", "should", "do", "does", "did", "can", "may", "might", - "must", "seem"}; - private static final Set AUX_VERBS = new HashSet(Arrays.asList(AUX_VERB_ARR)); - private static final String AUX_VERB_TAG = "VBD-AUX"; - - public static void initModels() { - if (pipeline != null) return; - Properties props = new Properties(); - props.put("annotators", Joiner.on(',').join(opts.annotators)); - if (opts.caseSensitive) { - props.put("pos.model", "edu/stanford/nlp/models/pos-tagger/english-bidirectional/english-bidirectional-distsim.tagger"); - props.put("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz,edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz"); - } else { - props.put("pos.model", "edu/stanford/nlp/models/pos-tagger/english-caseless-left3words-distsim.tagger"); - props.put("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz,edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"); - } - pipeline = new StanfordCoreNLP(props); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - LanguageInfo that = (LanguageInfo) o; - - if (!lemmaTokens.equals(that.lemmaTokens)) return false; - if (!nerTags.equals(that.nerTags)) return false; - if (!posTags.equals(that.posTags)) return false; - if (!tokens.equals(that.tokens)) return false; - - return true; + this.dependencyChildren = dependencyChildren; } // Return a string representing the tokens between start and end. @@ -124,9 +92,9 @@ public class LanguageInfo implements MemUsage.Instrumented { public String posSeq(int start, int end) { return sliceSequence(posTags, start, end); } - + public String canonicalPosSeq(int start, int end) { - if (start >= end) throw new RuntimeException("Bad indices, start="+start+", end="+end); + if (start >= end) throw new RuntimeException("Bad indices, start=" + start + ", end=" + end); if (end - start == 1) return LanguageUtils.getCanonicalPos(posTags.get(start)); StringBuilder out = new StringBuilder(); for (int i = start; i < end; i++) { @@ -142,7 +110,7 @@ public class LanguageInfo implements MemUsage.Instrumented { private static String sliceSequence(List items, int start, int end) { - if (start >= end) throw new RuntimeException("Bad indices, start="+start+", end="+end); + if (start >= end) throw new RuntimeException("Bad indices, start=" + start + ", end=" + end); if (end - start == 1) return items.get(start); StringBuilder out = new StringBuilder(); for (int i = start; i < end; i++) { @@ -152,96 +120,49 @@ public class LanguageInfo implements MemUsage.Instrumented { return out.toString(); } - public void analyze(String utterance) { - // Stanford tokenizer doesn't break hyphens. - // Replace hypens with spaces for utterances like - // "Spanish-speaking countries" but not for "2012-03-28". - StringBuilder buf = new StringBuilder(utterance); - for (int i = 0; i < buf.length(); i++) { - if (buf.charAt(i) == '-' && (i+1 < buf.length() && Character.isLetter(buf.charAt(i+1)))) - buf.setCharAt(i, ' '); - } - utterance = buf.toString(); - - // Clear these so that analyze can hypothetically be called - // multiple times. - tokens.clear(); - posTags.clear(); - nerTags.clear(); - nerValues.clear(); - lemmaTokens.clear(); - - if (opts.useAnnotators) { - initModels(); - Annotation annotation = pipeline.process(utterance); - - for (CoreLabel token : annotation.get(CoreAnnotations.TokensAnnotation.class)) { - String word = token.get(TextAnnotation.class); - String wordLower = word.toLowerCase(); - if (opts.caseSensitive) { - tokens.add(word); - } else { - tokens.add(wordLower); - } - posTags.add( - AUX_VERBS.contains(wordLower) ? - AUX_VERB_TAG : - token.get(PartOfSpeechAnnotation.class)); - nerTags.add(token.get(NamedEntityTagAnnotation.class)); - lemmaTokens.add(token.get(LemmaAnnotation.class)); - nerValues.add(token.get(NormalizedNamedEntityTagAnnotation.class)); - } - } else { - // Create tokens crudely - for (String token : utterance.trim().split("\\s+")) { - tokens.add(token); - lemmaTokens.add(token); - try { - Double.parseDouble(token); - posTags.add("CD"); - nerTags.add("NUMBER"); - nerValues.add(token); - } catch (NumberFormatException e ){ - posTags.add("UNK"); - nerTags.add("UNK"); - nerValues.add("UNK"); - } - } - } - } - // If all the tokens in [start, end) have the same nerValues, but not - // start-1 and end+1 (in other words, [start, end) is maximal), then return + // start - 1 and end + 1 (in other words, [start, end) is maximal), then return // the normalizedTag. Example: queryNerTag = "DATE". public String getNormalizedNerSpan(String queryTag, int start, int end) { String value = nerValues.get(start); + if (value == null) return null; if (!queryTag.equals(nerTags.get(start))) return null; - if (start-1 >= 0 && value.equals(nerValues.get(start-1))) return null; + if (start - 1 >= 0 && value.equals(nerValues.get(start - 1))) return null; if (end < nerValues.size() && value.equals(nerValues.get(end))) return null; - for (int i = start+1; i < end; i++) + for (int i = start + 1; i < end; i++) if (!value.equals(nerValues.get(i))) return null; + value = omitComparative(value); + return value; + } + + private String omitComparative(String value) { + if (value.startsWith("<=") || value.startsWith(">=")) + return value.substring(2); + if (value.startsWith("<") || value.startsWith(">")) + return value.substring(1); return value; } public String getCanonicalPos(int index) { + if (index == -1) return "OUT"; return LanguageUtils.getCanonicalPos(posTags.get(index)); } public boolean equalTokens(LanguageInfo other) { - if(tokens.size()!=other.tokens.size()) + if (tokens.size() != other.tokens.size()) return false; - for(int i = 0; i < tokens.size(); ++i) { - if(!tokens.get(i).equals(other.tokens.get(i))) + for (int i = 0; i < tokens.size(); ++i) { + if (!tokens.get(i).equals(other.tokens.get(i))) return false; } return true; } public boolean equalLemmas(LanguageInfo other) { - if(lemmaTokens.size()!=other.lemmaTokens.size()) + if (lemmaTokens.size() != other.lemmaTokens.size()) return false; - for(int i = 0; i < tokens.size(); ++i) { - if(!lemmaTokens.get(i).equals(other.lemmaTokens.get(i))) + for (int i = 0; i < tokens.size(); ++i) { + if (!lemmaTokens.get(i).equals(other.lemmaTokens.get(i))) return false; } return true; @@ -253,12 +174,12 @@ public class LanguageInfo implements MemUsage.Instrumented { public LanguageInfo remove(int startIndex, int endIndex) { - if(startIndex > endIndex || startIndex<0 || endIndex > numTokens()) - throw new RuntimeException("Illegal start or end index, start: " + startIndex + ", end: " + endIndex+", info size: " + numTokens()); + if (startIndex > endIndex || startIndex < 0 || endIndex > numTokens()) + throw new RuntimeException("Illegal start or end index, start: " + startIndex + ", end: " + endIndex + ", info size: " + numTokens()); LanguageInfo res = new LanguageInfo(); - for(int i = 0; i < numTokens(); ++i) { - if(i=endIndex) { + for (int i = 0; i < numTokens(); ++i) { + if (i < startIndex || i >= endIndex) { res.tokens.add(this.tokens.get(i)); res.lemmaTokens.add(this.lemmaTokens.get(i)); res.nerTags.add(this.nerTags.get(i)); @@ -270,7 +191,7 @@ public class LanguageInfo implements MemUsage.Instrumented { } public void addSpan(LanguageInfo other, int start, int end) { - for(int i = start; i < end; ++i) { + for (int i = start; i < end; ++i) { this.tokens.add(other.tokens.get(i)); this.lemmaTokens.add(other.lemmaTokens.get(i)); this.posTags.add(other.posTags.get(i)); @@ -279,11 +200,11 @@ public class LanguageInfo implements MemUsage.Instrumented { } } - public List getSpanProperties(int start, int end) { - List res =new ArrayList(); - res.add("lemmas="+lemmaPhrase(start, end)); - res.add("pos="+posSeq(start, end)); - res.add("ner="+nerSeq(start, end)); + public List getSpanProperties(int start, int end) { + List res = new ArrayList(); + res.add("lemmas=" + lemmaPhrase(start, end)); + res.add("pos=" + posSeq(start, end)); + res.add("ner=" + nerSeq(start, end)); return res; } @@ -296,7 +217,7 @@ public class LanguageInfo implements MemUsage.Instrumented { } public void addWordInfos(List wordInfos) { - for(WordInfo wInfo: wordInfos) + for (WordInfo wInfo : wordInfos) addWordInfo(wInfo); } @@ -310,29 +231,28 @@ public class LanguageInfo implements MemUsage.Instrumented { */ public Set getNamedEntitySpans() { Set res = new LinkedHashSet(); - int start=-1; + int start = -1; String prevTag = "O"; - for(int i = 0; i < nerTags.size(); ++i) { + for (int i = 0; i < nerTags.size(); ++i) { String currTag = nerTags.get(i); - if(currTag.equals("O")) { - if(!prevTag.equals("O")) { - res.add(new IntPair(start,i)); - start=-1; + if (currTag.equals("O")) { + if (!prevTag.equals("O")) { + res.add(new IntPair(start, i)); + start = -1; } - } - else { //currNe is not "O" - if(!currTag.equals(prevTag)) { - if(!prevTag.equals("O")) { - res.add(new IntPair(start,i)); + } else { // currNe is not "O" + if (!currTag.equals(prevTag)) { + if (!prevTag.equals("O")) { + res.add(new IntPair(start, i)); } - start=i; + start = i; } } prevTag = currTag; } - if(start!=-1) - res.add(new IntPair(start,nerTags.size())); + if (start != -1) + res.add(new IntPair(start, nerTags.size())); return res; } @@ -342,59 +262,57 @@ public class LanguageInfo implements MemUsage.Instrumented { */ public Set getProperNounSpans() { Set res = new LinkedHashSet(); - int start=-1; + int start = -1; String prevTag = "O"; - for(int i = 0; i < posTags.size(); ++i) { + for (int i = 0; i < posTags.size(); ++i) { String currTag = posTags.get(i); - if(LanguageUtils.isProperNoun(currTag)) { - if(!LanguageUtils.isProperNoun(prevTag)) - start=i; - } - else { //curr tag is not proper noun - if(LanguageUtils.isProperNoun(prevTag)) { - res.add(new IntPair(start,i)); - start=-1; + if (LanguageUtils.isProperNoun(currTag)) { + if (!LanguageUtils.isProperNoun(prevTag)) + start = i; + } else { // curr tag is not proper noun + if (LanguageUtils.isProperNoun(prevTag)) { + res.add(new IntPair(start, i)); + start = -1; } } prevTag = currTag; } - if(start!=-1) - res.add(new IntPair(start,posTags.size())); + if (start != -1) + res.add(new IntPair(start, posTags.size())); return res; } - + public Set getNamedEntitiesAndProperNouns() { Set res = getNamedEntitySpans(); res.addAll(getProperNounSpans()); return res; } - public Map getLemmaSpans() { - if(lemmaSpans==null) { - lemmaSpans = new HashMap(); - for(int i = 0; i < numTokens()-1; ++i) { - for(int j = i+1; j < numTokens(); ++j) - lemmaSpans.put(lemmaPhrase(i, j),new IntPair(i,j)); + public Map getLemmaSpans() { + if (lemmaSpans == null) { + lemmaSpans = new HashMap(); + for (int i = 0; i < numTokens() - 1; ++i) { + for (int j = i + 1; j < numTokens(); ++j) + lemmaSpans.put(lemmaPhrase(i, j), new IntPair(i, j)); } } return lemmaSpans; } - + public boolean matchLemmas(List wordInfos) { - for(int i = 0; i < numTokens(); ++i) { - if(matchLemmasFromIndex(wordInfos,i)) + for (int i = 0; i < numTokens(); ++i) { + if (matchLemmasFromIndex(wordInfos, i)) return true; } return false; } private boolean matchLemmasFromIndex(List wordInfos, int start) { - - if(start+wordInfos.size()>numTokens()) + if (start + wordInfos.size() > numTokens()) return false; - for(int j = 0; j < wordInfos.size();++j) { - if(!wordInfos.get(j).lemma.equals(lemmaTokens.get(start+j))) + for (int j = 0; j < wordInfos.size(); ++j) { + if (!wordInfos.get(j).lemma.equals(lemmaTokens.get(start + j))) return false; } return true; @@ -407,8 +325,8 @@ public class LanguageInfo implements MemUsage.Instrumented { */ public static class LanguageUtils { public static boolean sameProperNounClass(String noun1, String noun2) { - if(( noun1.equals("NNP")|| noun1.equals("NNPS")) && - ( noun2.equals("NNP")|| noun2.equals("NNPS"))) + if ((noun1.equals("NNP") || noun1.equals("NNPS")) && + (noun2.equals("NNP") || noun2.equals("NNPS"))) return true; return false; } @@ -431,8 +349,8 @@ public class LanguageInfo implements MemUsage.Instrumented { public static String getLemmaPhrase(List wordInfos) { String[] res = new String[wordInfos.size()]; - for(int i = 0; i < wordInfos.size(); ++i) { - res[i]=wordInfos.get(i).lemma; + for (int i = 0; i < wordInfos.size(); ++i) { + res[i] = wordInfos.get(i).lemma; } return Joiner.on(' ').join(res); } @@ -447,15 +365,21 @@ public class LanguageInfo implements MemUsage.Instrumented { @Override public long getBytes() { - return MemUsage.objectSize(MemUsage.pointerSize*2)+MemUsage.getBytes(tokens)+MemUsage.getBytes(lemmaTokens) - +MemUsage.getBytes(posTags)+MemUsage.getBytes(nerTags)+MemUsage.getBytes(nerValues) - +MemUsage.getBytes(lemmaSpans); + return MemUsage.objectSize(MemUsage.pointerSize * 2) + MemUsage.getBytes(tokens) + MemUsage.getBytes(lemmaTokens) + + MemUsage.getBytes(posTags) + MemUsage.getBytes(nerTags) + MemUsage.getBytes(nerValues) + + MemUsage.getBytes(lemmaSpans); } - + public boolean isNumberAndDate(int index) { return posTags.get(index).equals("CD") && nerTags.get(index).equals("DATE"); } + public static boolean isContentWord(String pos) { + return pos.equals("NN") || pos.equals("NNS") || + (pos.startsWith("V") && !pos.equals("VBD-AUX")) || + pos.startsWith("J"); + } + public static class WordInfo { public final String token; public final String lemma; @@ -463,7 +387,7 @@ public class LanguageInfo implements MemUsage.Instrumented { public final String nerTag; public final String nerValue; public WordInfo(String token, String lemma, String pos, String nerTag, String nerValue) { - this.token = token; this.lemma=lemma; this.pos = pos; this.nerTag=nerTag; this.nerValue=nerValue; + this.token = token; this.lemma = lemma; this.pos = pos; this.nerTag = nerTag; this.nerValue = nerValue; } public String toString() { diff --git a/src/edu/stanford/nlp/sempre/Learner.java b/src/edu/stanford/nlp/sempre/Learner.java index ae45066..9a89938 100644 --- a/src/edu/stanford/nlp/sempre/Learner.java +++ b/src/edu/stanford/nlp/sempre/Learner.java @@ -6,7 +6,10 @@ import fig.basic.*; import fig.exec.Execution; import java.io.PrintWriter; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * The main learning loop. Goes over a dataset multiple times, calling the @@ -18,85 +21,79 @@ public class Learner { public static class Options { @Option(gloss = "Number of iterations to train") public int maxTrainIters = 0; + @Option(gloss = "When using mini-batch updates for SGD, this is the batch size") public int batchSize = 1; // Default is SGD - @Option(gloss = "Don't update feature weights that start with this") - public List staticFeaturePrefixes = new ArrayList(); + @Option(gloss = "Write predDerivations to examples file (huge)") public boolean outputPredDerivations = false; - @Option(gloss = "Multiply beam size by this factor after each iteration") - public double beamSizeIncreaseFactor = 1; - @Option(gloss = "Take beams of this size in each iteration (repeating the " + - "last value. Overrides beamSizeIncreaseFactor when set.") - public List beamSizePerIteration = null; + @Option(gloss = "Dump all features and compatibility scores") public boolean dumpFeaturesAndCompatibility = false; - @Option(gloss = "Always update when using listwise model.") - public boolean alwaysUpdate = false; - @Option(gloss = "Whether to update based on partial reward.") - public boolean partialReward = false; + @Option(gloss = "Whether to add feedback") + public boolean addFeedback = false; + @Option(gloss = "Whether to sort on feedback") + public boolean sortOnFeedback = true; - // DELETE? - @Option(gloss = "Use binary logistic regression model") - public boolean binaryLogistic = false; - @Option(gloss = "Only update weights when beam has at least one correct prediction.") - public boolean binaryLogisticThrottledUpdates = false; + @Option(gloss = "Verbosity") public int verbose = 0; + + @Option(gloss = "Initialize with these parameters") + public List> initialization; + + @Option(gloss = "whether to update weights") + public boolean updateWeights = true; + @Option(gloss = "whether to check gradient") + public boolean checkGradient = false; } + public static Options opts = new Options(); - private static final double EPSILON = 0.000000001d; - - private final Parser parser; + private Parser parser; private final Params params; private final Dataset dataset; - private final FeatureMatcher updateFeatureMatcher; // Update parameters for features that match according to this. private final PrintWriter eventsOut; // For printing a machine-readable log - - public static Options opts = new Options(); + private final List semFuncsToUpdate; public Learner(Parser parser, Params params, Dataset dataset) { this.parser = parser; this.params = params; this.dataset = dataset; this.eventsOut = IOUtils.openOutAppendEasy(Execution.getFile("learner.events")); + if (opts.initialization != null) + this.params.init(opts.initialization); + // collect all semantic functions to update + semFuncsToUpdate = new ArrayList<>(); - // Only update features that don't match any of the static feature prefixes. - this.updateFeatureMatcher = new FeatureMatcher() { - @Override - public boolean matches(String feature) { - for (String prefix : opts.staticFeaturePrefixes) - if (feature.startsWith(prefix)) - return false; - return true; + for (Rule rule : parser.grammar.getRules()) { + SemanticFn currSemFn = rule.getSem(); + boolean toAdd = true; + for (SemanticFn semFuncToUpdate : semFuncsToUpdate) { + if (semFuncToUpdate.getClass().equals(currSemFn.getClass())) { + toAdd = false; + break; + } } - }; + if (toAdd) + semFuncsToUpdate.add(currSemFn); + } } public void learn() { - learn(-1); - } - - public void learn(int iters) { - Map> m = Maps.newHashMap(); - learn(iters, m); - } - - public void learn(Map> evaluations) { - learn(-1, evaluations); + learn(opts.maxTrainIters, Maps.newHashMap()); } /** * @param evaluations Evaluations per iteration per group. */ - public void learn(int iters, Map> evaluations) { + public void learn(int numIters, Map> evaluations) { LogInfo.begin_track("Learner.learn()"); - - if (iters < 0) - iters = opts.maxTrainIters; - + // if when we start we have parameters already - need to sort the semantic functions. + if (!params.isEmpty()) + sortOnFeedback(); // For each iteration, go through the groups and parse (updating if train). - for (int iter = 0; iter <= iters; iter++) { - LogInfo.begin_track("Iteration %s/%s", iter, iters); + for (int iter = 0; iter <= numIters; iter++) { + + LogInfo.begin_track("Iteration %s/%s", iter, numIters); Execution.putOutput("iter", iter); // Averaged over all iterations @@ -109,14 +106,13 @@ public class Learner { // Test and train for (String group : dataset.groups()) { - boolean lastIter = iter == iters; - boolean updateWeights = group.equals("train") && !lastIter; // Don't train on last iteration + boolean lastIter = (iter == numIters); + boolean updateWeights = opts.updateWeights && group.equals("train") && !lastIter; // Don't train on last iteration Evaluation eval = processExamples( - iter, - group, - dataset.examples(group), - updateWeights, - lastIter); + iter, + group, + dataset.examples(group), + updateWeights); MapUtils.addToList(evaluations, group, eval); meanEvaluations.get(group).add(eval); StopWatchSet.logStats(); @@ -130,96 +126,28 @@ public class Learner { } // Write out examples and predictions - for (String group : dataset.groups()) - Vis.writeExamples(iter, group, dataset.examples(group), opts.outputPredDerivations); + if (opts.outputPredDerivations) + for (String group : dataset.groups()) + ExampleUtils.writeSDF(iter, group, meanEvaluations.get(group), dataset.examples(group), opts.outputPredDerivations); LogInfo.end_track(); } - LogInfo.logs("Learner: number of touched binaries: %s",FbFormulasInfo.numOfTouchedBinaries()); LogInfo.end_track(); } public void onlineLearnExample(Example ex) { LogInfo.begin_track("onlineLearnExample: %s derivations", ex.predDerivations.size()); - HashMap counts = new HashMap(); - updateCounts(ex, counts); + HashMap counts = new HashMap<>(); + for (Derivation deriv : ex.predDerivations) + deriv.compatibility = parser.valueEvaluator.getCompatibility(ex.targetValue, deriv.value); + ParserState.computeExpectedCounts(ex.predDerivations, counts); params.update(counts); LogInfo.end_track(); } - // How much reward do we get? - private double reward(Value targetValue, Value predValue) { - if (targetValue != null) { - if(opts.partialReward) - return Math.log(targetValue.getCompatibility(predValue)); - return Math.log(targetValue.getCompatibility(predValue) == 1 ? 1 : 0); - } - return 0; - } - - private void computeExpectedCounts(Example ex, List derivations, Map counts) { - final boolean addDummy = opts.alwaysUpdate && !opts.binaryLogistic; - - double[] trueScores; - double[] predScores; - - int n = derivations.size(); - if (n == 0) - return; - - trueScores = new double[n + (addDummy ? 1 : 0)]; - predScores = new double[n + (addDummy ? 1 : 0)]; - for (int i = 0; i < n; i++) { - Derivation deriv = derivations.get(i); - double r = reward(ex.targetValue, deriv.value); - if (opts.binaryLogistic) { - trueScores[i] = Math.exp(r); - predScores[i] = 1.0d / (1.0d + Math.exp(-deriv.score)); - } else { - trueScores[i] = deriv.score + r; - predScores[i] = deriv.score; - } - } - if (addDummy) { - trueScores[n] = Math.log(EPSILON); - predScores[n] = 0.0d; - } - - if (!opts.binaryLogistic) { - if (!NumUtils.expNormalize(trueScores)) return; - if (!NumUtils.expNormalize(predScores)) return; - } else if (opts.binaryLogisticThrottledUpdates) { - // Only update if there's a correct prediction on the beam, so - // as to provide comparison with non-binary model. - boolean ok = false; - for (int i = 0; i < n; i++) { - if (trueScores[i] > 0.0d) { - ok = true; - break; - } - } - if (!ok) - return; - } - //LogInfo.logs("TRUE: %s", Fmt.D(trueScores)); - //LogInfo.logs("PRED: %s", Fmt.D(predScores)); - - // Update parameters - for (int i = 0; i < n; i++) { - Derivation deriv = derivations.get(i); - double incr = trueScores[i] - predScores[i]; - //LogInfo.logs("incr=%s, feature=%s",incr,deriv.getAllFeatureVector().get("alignmentScores :: binary.top")); - deriv.incrementAllFeatureVector(incr, counts, updateFeatureMatcher); - } - //LogInfo.logs("Gradient=%s",counts); - } - private Evaluation processExamples(int iter, String group, List examples, - boolean doUpdateWeights, - boolean lastIter) { - setBeamSizeOnExamples(examples, iter, lastIter); - + boolean computeExpectedCounts) { Evaluation evaluation = new Evaluation(); if (examples.size() == 0) @@ -229,22 +157,30 @@ public class Learner { Execution.putOutput("group", group); LogInfo.begin_track_printAll( - "Processing %s: %s examples", prefix, examples.size()); + "Processing %s: %s examples", prefix, examples.size()); LogInfo.begin_track("Examples"); - Map counts = new HashMap(); + Map counts = new HashMap<>(); int batchSize = 0; for (int e = 0; e < examples.size(); e++) { + Example ex = examples.get(e); LogInfo.begin_track_printAll( - "%s: example %s/%s: %s", prefix, e, examples.size(), ex.id); + "%s: example %s/%s: %s", prefix, e, examples.size(), ex.id); ex.log(); Execution.putOutput("example", e); - parseExample(ex, true); - if (doUpdateWeights) { - updateCounts(ex, counts); + ParserState state = parseExample(params, ex, computeExpectedCounts); + if (computeExpectedCounts) { + if (opts.checkGradient) { + LogInfo.begin_track("Checking gradient"); + checkGradient(ex, state); + LogInfo.end_track(); + } + + SempreUtils.addToDoubleMap(counts, state.expectedCounts); + batchSize++; if (batchSize >= opts.batchSize) { // Gathered enough examples, update parameters @@ -252,15 +188,26 @@ public class Learner { batchSize = 0; } } - logExampleEvaluation(ex); - accumulateAndLogEvaluation(ex, evaluation, prefix); + // } + + LogInfo.logs("Current: %s", ex.evaluation.summary()); + evaluation.add(ex.evaluation); + LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary()); + printLearnerEventsIter(ex, iter, group); LogInfo.end_track(); + if (opts.addFeedback && computeExpectedCounts) + addFeedback(ex); + + // To save memory ex.predDerivations.clear(); - ex.predDerivationsAfterParse.clear(); } - if (doUpdateWeights && batchSize > 0) + + if (computeExpectedCounts && batchSize > 0) updateWeights(counts); + params.finalizeWeights(); + if (opts.sortOnFeedback && computeExpectedCounts) + sortOnFeedback(); LogInfo.end_track(); logEvaluationStats(evaluation, prefix); @@ -269,64 +216,63 @@ public class Learner { return evaluation; } - private void setBeamSizeOnExamples(List examples, - int iter, - boolean lastIter) { - for (Example ex : examples) { - // Set beam size - if (ex.beamSize == -1) { - ex.beamSize = parser.getDefaultBeamSize(); - } else if (ex.correctMaxBeamPosition != -1) { - //ex.beamSize = (ex.correctMaxBeamPosition+1) * 2; - } else if (!lastIter) { - // Increase the beam to try to get something correct. - if (opts.beamSizePerIteration != null) { - int i = Math.min(iter, opts.beamSizePerIteration.size() - 1); - ex.beamSize = opts.beamSizePerIteration.get(i); - } else { - ex.beamSize *= opts.beamSizeIncreaseFactor; - } - } + private void checkGradient(Example ex, ParserState state) { + double eps = 1e-2; + for (String feature : state.expectedCounts.keySet()) { + LogInfo.begin_track("feature=%s", feature); + double computedGradient = state.expectedCounts.get(feature); + Params perturbedParams = this.params.copyParams(); + perturbedParams.getWeights().put(feature, perturbedParams.getWeight(feature) + eps); + ParserState perturbedState = parseExample(perturbedParams, ex, true); + double checkedGradient = (perturbedState.objectiveValue - state.objectiveValue) / eps; + LogInfo.logs("Learner.checkGradient(): weight=%s, pertWeight=%s, obj=%s, pertObj=%s, feature=%s, computed=%s, checked=%s, diff=%s", + params.getWeight(feature), perturbedParams.getWeight(feature), + state.objectiveValue, perturbedState.objectiveValue, + feature, + computedGradient, checkedGradient, Math.abs(checkedGradient - computedGradient)); + LogInfo.end_track(); } } - private void parseExample(Example ex, boolean execAllowed) { - StopWatchSet.begin("Parser.parse"); - parser.parse(params, ex, execAllowed); - StopWatchSet.end(); + private void sortOnFeedback() { + for (SemanticFn semFn : semFuncsToUpdate) { + semFn.sortOnFeedback(parser.getSearchParams(params)); + } } - private void updateCounts(Example ex, Map counts) { - computeExpectedCounts(ex, ex.predDerivations, counts); + private void addFeedback(Example ex) { + for (SemanticFn semFn : semFuncsToUpdate) { + semFn.addFeedback(ex); + } + } + + private ParserState parseExample(Params params, Example ex, boolean computeExpectedCounts) { + StopWatchSet.begin("Parser.parse"); + ParserState res = this.parser.parse(params, ex, computeExpectedCounts); + StopWatchSet.end(); + return res; } private void updateWeights(Map counts) { StopWatchSet.begin("Learner.updateWeights"); - LogInfo.begin_track("Updating weights"); + LogInfo.begin_track("Updating learner weights"); double sum = 0; for (double v : counts.values()) sum += v * v; + if (opts.verbose >= 2) + SempreUtils.logMap(counts, "gradient"); LogInfo.logs("L2 norm: %s", Math.sqrt(sum)); params.update(counts); + if (opts.verbose >= 2) + params.log(); counts.clear(); LogInfo.end_track(); StopWatchSet.end(); } - private void logExampleEvaluation(Example ex) { - LogInfo.logs("Current: %s", ex.computeTotalEvaluation().summary()); - } - - private void accumulateAndLogEvaluation(Example ex, - Evaluation evaluation, - String prefix) { - evaluation.add(ex.computeTotalEvaluation()); - LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary()); - } - // Print summary over all examples private void logEvaluationStats(Evaluation evaluation, String prefix) { LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary()); - evaluation.add(LexiconFn.lexEval); + // evaluation.add(LexiconFn.lexEval); evaluation.logStats(prefix); evaluation.putOutput(prefix); } @@ -334,7 +280,7 @@ public class Learner { private void printLearnerEventsIter(Example ex, int iter, String group) { if (eventsOut == null) return; - List fields = new ArrayList(); + List fields = new ArrayList<>(); fields.add("iter=" + iter); fields.add("group=" + group); fields.add("utterance=" + ex.utterance); @@ -344,18 +290,18 @@ public class Learner { fields.add("predValue=" + deriv.value); fields.add("predFormula=" + deriv.formula); } - fields.add(ex.computeTotalEvaluation().summary("\t")); + fields.add(ex.evaluation.summary("\t")); eventsOut.println(Joiner.on('\t').join(fields)); eventsOut.flush(); // Print out features and the compatibility across all the derivations if (opts.dumpFeaturesAndCompatibility) { for (Derivation deriv : ex.predDerivations) { - fields = new ArrayList(); + fields = new ArrayList<>(); fields.add("iter=" + iter); fields.add("group=" + group); fields.add("utterance=" + ex.utterance); - Map features = new HashMap(); + Map features = new HashMap<>(); deriv.incrementAllFeatureVector(1, features); for (String f : features.keySet()) { double v = features.get(f); @@ -372,27 +318,11 @@ public class Learner { String group) { if (eventsOut == null) return; - List fields = new ArrayList(); + List fields = new ArrayList<>(); fields.add("iter=" + iter); fields.add("group=" + group); fields.add(evaluation.summary("\t")); eventsOut.println(Joiner.on('\t').join(fields)); eventsOut.flush(); } -/* - private void l2Reg(Map counts) { - Set features = new HashSet(params.weights.keySet()); - features.addAll(counts.keySet()); - for (String feature : features) { - MapUtils.incr(counts, feature, opts.l2RegCoefficient * -params.getWeight(feature)); - } - } - - private void l1Reg(Map counts) { - Set features = new HashSet(params.weights.keySet()); - features.addAll(counts.keySet()); - for (String feature : features) { - MapUtils.incr(counts, feature, opts.l1RegCoefficient * - Math.signum(params.getWeight(feature))); - } - }*/ } diff --git a/src/edu/stanford/nlp/sempre/LexiconFn.java b/src/edu/stanford/nlp/sempre/LexiconFn.java deleted file mode 100644 index 0fe4afe..0000000 --- a/src/edu/stanford/nlp/sempre/LexiconFn.java +++ /dev/null @@ -1,334 +0,0 @@ -package edu.stanford.nlp.sempre; - -import com.google.common.base.Joiner; -import edu.stanford.nlp.sempre.fbalignment.lexicons.EntrySource; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.BinaryLexicalEntry; -import edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.EntityLexicon; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import fig.basic.*; - -import org.apache.lucene.queryparser.classic.ParseException; - -import java.io.IOException; -import java.util.*; - -/** - * Looks up a string into the lexicon, interfacing with the fbalignment code. - * @author Percy Liang - */ -public class LexiconFn extends SemanticFn { - public static class Options { - @Option(gloss = "Keep entries with at most this distance") - public double maxDistance = 100; - @Option(gloss = "Keep entries with at most this distance") - public double entityMaxDistance = Integer.MAX_VALUE; - @Option(gloss = "Keep entries with at least this popularity") - public double minPopularity = 0; - @Option(gloss = "Number of entities to return from entity lexicon") - public int maxEntityEntries = 100; //we filter here and not in entity lexicon so that don't need different cache for different numbers - @Option(gloss = "Verbose") public int verbose = 0; - @Option(gloss = "Class name for lexicon") public String lexiconClassName; - } - - public static Options opts = new Options(); - - private static Lexicon lexicon; - public static Evaluation lexEval = new Evaluation(); - - private String mode; // unary, binary, or entity - private EntityLexicon.SearchStrategy entitySearchStrategy = Lexicon.opts.entitySearchStrategy; // For entities, how to search - private TextToTextMatcher textToTextMatcher = new TextToTextMatcher(); - private FbFormulasInfo fbFormulaInfo; - - public LexiconFn() throws IOException { - lexicon = Lexicon.getSingleton(); - fbFormulaInfo = FbFormulasInfo.getSingleton(); - } - - public void init(LispTree tree) { - super.init(tree); - for (int i = 1; i < tree.children.size(); i++) { - String value = tree.child(i).value; - if (value.equals("unary")) this.mode = "unary"; - else if (value.equals("binary")) this.mode = "binary"; - else if (value.equals("entity")) this.mode = "entity"; - else if (value.equals("allowInexact")/*deprecate this*/ || value.equals("inexact")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.inexact; - else if (value.equals("fbsearch")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.fbsearch; - else throw new RuntimeException("Invalid argument: " + value); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - LexiconFn lexiconFn = (LexiconFn) o; - if (!mode.equals(lexiconFn.mode)) return false; - return true; - } - - boolean shouldKeep(LexicalEntry entry) { - if (entry.getPopularity() < opts.minPopularity) return false; - if (entry.getDistance() > opts.maxDistance) return false; - if (entry instanceof LexicalEntry.EntityLexicalEntry && entry.getDistance() > opts.entityMaxDistance) - return false; - return true; - } - - // Input: set of types coming from the lexicon {fb:common.topic, fb:people.person, ...} - // Output: remove any element which is in the transitive closure. {fb:people.person, ...} - private UnionSemType setToType(Set types) { - FbFormulasInfo info = FbFormulasInfo.getSingleton(); - Set resultTypes = new HashSet(types); - for (String entityType : types) { - for (String supertype : info.getIncludedTypesInclusive(entityType)) { - if (!supertype.equals(entityType)) - resultTypes.remove(supertype); - } - } - UnionSemType type = new UnionSemType(); - for (String entityType : resultTypes) - type.add(new EntitySemType(entityType)); - return type; - } - - // Convert LexicalEntry into a form consumable by the semantic parser. - private Derivation convert(Example ex, - Callable c, - String mode, - String word, - LexicalEntry entry, - Counter> typesThatAlreadyAppeared) { - FeatureVector features = new FeatureVector(); - SemType type; - - if (FeatureExtractor.containsDomain("basicStats")) { - features.addWithBias("basicStats", mode + ".popularity", Math.log(entry.getPopularity() + 1)); - //features.addWithBias(mode + ".distance", entry.getDistance()); - } - - if (mode.equals("entity")) { - // Entities - LexicalEntry.EntityLexicalEntry eEntry = (LexicalEntry.EntityLexicalEntry) entry; - - if (FeatureExtractor.containsDomain("tokensDistance")) { - for (String feature : eEntry.tokenEditDistanceFeatures.keySet()) { - double value = eEntry.tokenEditDistanceFeatures.getCount(feature); - features.addWithBias("tokensDistance", mode + "." + feature, value); - } - } - - type = setToType(((LexicalEntry.EntityLexicalEntry) entry).getTypes()); - } else if (mode.equals("unary")) { - // Unaries - LexicalEntry.UnaryLexicalEntry uEntry = (LexicalEntry.UnaryLexicalEntry) entry; - - // Alignment scores features - if (FeatureExtractor.containsDomain("alignmentScores")) { - for (String feature : uEntry.alignmentScores.keySet()) { - features.addWithBias("alignmentScores", mode + "." + feature, Math.log(MapUtils.getDouble(uEntry.alignmentScores, feature, 0.0) + 1)); - } - } - - if (FeatureExtractor.containsDomain("basicStats")) { - if (uEntry.getDistance() < 0.0001) - features.add("basicStats", "unary.equal"); - - //features.addWithBias(mode + ".distance", entry.getDistance()); - - //adding the source of the lexical entry as a feature - features.add("basicStats", mode + ",source=" + uEntry.source); - } - - type = setToType(((LexicalEntry.UnaryLexicalEntry) entry).getTypes()); - } else if (mode.equals("binary")) { - // Binaries - LexicalEntry.BinaryLexicalEntry bEntry = (LexicalEntry.BinaryLexicalEntry) entry; - - // Alignment scores features - if (FeatureExtractor.containsDomain("alignmentScores")) { - for (String feature : bEntry.alignmentScores.keySet()) { - //HACK for graph prop - if (bEntry.source == EntrySource.GRAPHPROP) - features.addWithBias("alignmentScores", mode + "." + feature, Math.log(MapUtils.getDouble(bEntry.alignmentScores, feature, 0.0))); - else - features.addWithBias("alignmentScores", mode + "." + feature, Math.log(MapUtils.getDouble(bEntry.alignmentScores, feature, 0.0) + 1)); - } - - //adding a feature if this is the top ranked alignment for this pair of types - //this assumes that the entries are passed sorted!!!! - // TODO: this seems dangerous - Pair expectedTypes = new Pair(bEntry.expectedType1, bEntry.expectedType2); - if (typesThatAlreadyAppeared.getCount(expectedTypes) < 1) { - features.add("alignmentScores", "binary.top"); - typesThatAlreadyAppeared.incrementCount(expectedTypes); - } - } - - if (FeatureExtractor.containsDomain("basicStats")) { - //adding the source of the lexical entry as a feature - features.add("basicStats", mode + "." + bEntry.source); - - //adding edit distance feature - replaced by text to text matcher - //features.addWithBias(mode + ".distance", entry.getDistance()); - } - - //text to text match features - features.add( - textToTextMatcher.extractFeatures( - ex.languageInfo.tokens.subList(c.getStart(), c.getEnd()), - ex.languageInfo.posTags.subList(c.getStart(), c.getEnd()), - ex.languageInfo.lemmaTokens.subList(c.getStart(), c.getEnd()), - bEntry.fbDescriptions)); - - // Note that expectedType2 is the argument type, expectedType1 is the return type. - type = new FuncSemType(new EntitySemType(bEntry.getExpectedType2()), new EntitySemType(bEntry.getExpectedType1())); - } else { - throw new RuntimeException("Invalid mode: " + mode); - } - - Derivation newDeriv = new Derivation.Builder() - .withCallable(c) - .formula(entry.formula) - .type(type) - .localFeatureVector(features) - .createDerivation(); - - // Doesn't generalize, but add it for now, otherwise not separable - if (FeatureExtractor.containsDomain("lexAlign")) - newDeriv.addFeature("lexAlign", word + " --- " + newDeriv.formula); - - if (SemanticFn.opts.trackLocalChoices) - newDeriv.localChoices.add("LexiconFn " + newDeriv.startEndString(ex.getTokens()) + " " + entry); - - if (opts.verbose >= 3) { - LogInfo.logs( - "LexiconFn: %s [%s => %s ~ %s | %s]: popularity = %s, distance = %s, type = %s, source=%s", - mode, word, entry.normalizedTextDesc, entry.fbDescriptions, newDeriv.formula, - entry.getPopularity(), entry.getDistance(), newDeriv.type,entry.source); - } - return newDeriv; - } - - public List call(Example ex, Callable c) { - if (opts.verbose >= 2) - LogInfo.begin_track("LexicalFn.call: %s", c.childStringValue(0)); - - String phrase = c.childStringValue(0); - List results = new ArrayList(); - - try { - // Entities - if (mode.equals("entity")) { - List entries = lexicon.lookupEntities(phrase, entitySearchStrategy); - if (opts.verbose >= 2) - LogInfo.logs("LexiconFn.call %s: %s => %s entries", mode, phrase, entries.size()); - - lexEval.add("entity", entries.isEmpty() ? false : true); - entries = entries.subList(0, Math.min(opts.maxEntityEntries, entries.size())); - - for (LexicalEntry entry : entries) - results.add(convert(ex, c, "entity", phrase, entry, null)); - } - - // Unaries - if (mode.equals("unary")) { - List entries = lexicon.lookupUnaryPredicates(phrase); - if (opts.verbose >= 2) - LogInfo.logs("LexiconFn.call %s: %s => %s entries", mode, phrase, entries.size()); - lexEval.add("unary", entries.isEmpty() ? false : true); - for (LexicalEntry entry : entries) - results.add(convert(ex, c, "unary", phrase, entry, null)); - } - - // Binaries - if (mode.equals("binary")) { - List entries = lexicon.lookupBinaryPredicates(phrase); - if (opts.verbose >= 2) - LogInfo.logs("LexiconFn.call %s: %s => %s entries", mode, phrase, entries.size()); - lexEval.add("binary", entries.isEmpty() ? false : true); - - //this set will contain pairs of types that appeared, each time a new one appears - //it is guaranteed it is the top one for this pair of types since the list is ordered - //THIS ASSUMES THAT THE ENTRIES ARE SORTED - Counter> typesThatAlreadyAppeared = new ClassicCounter>(); - - for (LexicalEntry entry : entries) { - //while the lexicon has CVT entries we just ignore them - if (fbFormulaInfo.isCvt(((BinaryLexicalEntry) entry).expectedType1) || fbFormulaInfo.isCvt(((BinaryLexicalEntry) entry).expectedType2)) - continue; - - Derivation deriv = convert(ex, c, "binary", phrase, entry, typesThatAlreadyAppeared); - // add context matching feature - if (FeatureExtractor.containsDomain("context")) { - if (!doesContextMatch(ex, deriv, (BinaryLexicalEntry) entry, phrase)) - deriv.addFeature("context", "binary.contextMismatch"); - } - results.add(deriv); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } catch (ParseException e) { - throw new RuntimeException(e); - } - - if (opts.verbose >= 2) LogInfo.end_track(); - - return results; - } - - /** For now this ignores stemming!!! */ - private static boolean doesContextMatch(Example ex, Derivation deriv, BinaryLexicalEntry bEntry, String word) { - - if (bEntry.isFullLexemeEqualToNormalizedText()) - return true; - //get the left and right context surrounding the core (normalized text) - String[] leftContext = bEntry.getLeftContext(); - String[] rightContext = bEntry.getRightContext(); - //match right context - for (int i = 0; i < rightContext.length; ++i) { - //in this case all context words were matched and some were dropped but there was no mismatch - if (deriv.end + i >= ex.numTokens() || ex.token(deriv.end + i).equals("?")) - break; - - if (!rightContext[0].equals(ex.lemmaToken(deriv.end + i))) { - if (opts.verbose >= 1) { - LogInfo.logs( - "RIGHT CONTEXT MISMATCH: full lexeme=%s, normalized text=%s left context=%s, right context=%s example=%s, formula=%s", - bEntry.fullLexeme, - bEntry.normalizedTextDesc, - Joiner.on(' ').join(leftContext), - Joiner.on(' ').join(rightContext), - Joiner.on(' ').join(ex.languageInfo.tokens), - bEntry.formula); - } - return false; - } - } - - //match right context - for (int i = 0; i < leftContext.length; ++i) { - if (deriv.start - i - 1 < 0) //in this case all context words were matched and some were dropped but there was no mismatch - break; - if (!leftContext[leftContext.length - i - 1].equals(ex.lemmaToken(deriv.start - i - 1))) { - if (opts.verbose >= 1) { - LogInfo.logs( - "LEFT CONTEXT MISMATCH: full lexeme=%s, normalized text=%s left context=%s, right context=%s example=%s, formula=%s", - bEntry.fullLexeme, - bEntry.normalizedTextDesc, - Joiner.on(' ').join(leftContext), - Joiner.on(' ').join(rightContext), - Joiner.on(' ').join(ex.languageInfo.tokens), - bEntry.formula); - } - return false; - } - } - return true; - } -} - diff --git a/src/edu/stanford/nlp/sempre/ListValue.java b/src/edu/stanford/nlp/sempre/ListValue.java index 40c0a00..07b37e1 100644 --- a/src/edu/stanford/nlp/sempre/ListValue.java +++ b/src/edu/stanford/nlp/sempre/ListValue.java @@ -1,6 +1,7 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; +import fig.basic.LogInfo; import java.util.ArrayList; import java.util.List; @@ -20,53 +21,13 @@ public class ListValue extends Value { LispTree tree = LispTree.proto.newList(); tree.addChild("list"); for (Value value : values) - tree.addChild(value.toLispTree()); + tree.addChild(value == null ? LispTree.proto.newLeaf(null) : value.toLispTree()); return tree; } - // Compute F1 score between two lists (partial match). - // this is target, that is predicted. - public double getCompatibility(Value thatValue) { - if (!(thatValue instanceof ListValue)) - return 0; - ListValue that = (ListValue) thatValue; - - if (this.values.size() == 0 && that.values.size() == 0) - return 1; - if (this.values.size() == 0 || that.values.size() == 0) - return 0; - - double precision = 0; - for (Value v2 : that.values) { // For every predicted value... - double score = 0; - for (Value v1 : this.values) - score = Math.max(score, v1.getCompatibility(v2)); - precision += score; - } - precision /= that.values.size(); - assert precision >= 0 && precision <= 1 : precision; - - double recall = 0; - for (Value v1 : this.values) { // For every true value... - double score = 0; - for (Value v2 : that.values) - score = Math.max(score, v1.getCompatibility(v2)); - recall += score; - } - recall /= this.values.size(); - assert recall >= 0 && recall <= 1 : recall; - - if (precision + recall == 0) return 0; - - double f1 = 2 * precision * recall / (precision + recall); - assert f1 >= 0 && f1 <= 1 : f1; - - return f1; - } - public void log() { for (Value value : values) - value.log(); + LogInfo.logs("%s", value); } @Override diff --git a/src/edu/stanford/nlp/sempre/Main.java b/src/edu/stanford/nlp/sempre/Main.java index 460bc8e..c150617 100644 --- a/src/edu/stanford/nlp/sempre/Main.java +++ b/src/edu/stanford/nlp/sempre/Main.java @@ -10,6 +10,7 @@ import fig.exec.Execution; */ public class Main implements Runnable { @Option public boolean interactive = false; + @Option public boolean server = false; public void run() { Builder builder = new Builder(); @@ -21,6 +22,12 @@ public class Main implements Runnable { Learner learner = new Learner(builder.parser, builder.params, dataset); learner.learn(); + if (server) { + Master master = new Master(builder); + Server server = new Server(master); + server.run(); + } + if (interactive) { Master master = new Master(builder); master.runInteractivePrompt(); diff --git a/src/edu/stanford/nlp/sempre/MarkFormula.java b/src/edu/stanford/nlp/sempre/MarkFormula.java index b770688..e5ad4b3 100644 --- a/src/edu/stanford/nlp/sempre/MarkFormula.java +++ b/src/edu/stanford/nlp/sempre/MarkFormula.java @@ -3,6 +3,8 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * Usage: * (mark |var| |body|). @@ -36,13 +38,21 @@ public class MarkFormula extends Formula { return result == null ? new MarkFormula(var, body.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) + res.addAll(body.mapToList(func, alwaysRecurse)); + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof MarkFormula)) return false; MarkFormula that = (MarkFormula) thatObj; return this.var.equals(that.var) && this.body.equals(that.body); } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + var.hashCode(); diff --git a/src/edu/stanford/nlp/sempre/Master.java b/src/edu/stanford/nlp/sempre/Master.java index 496e109..eaa211d 100644 --- a/src/edu/stanford/nlp/sempre/Master.java +++ b/src/edu/stanford/nlp/sempre/Master.java @@ -3,16 +3,12 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import edu.stanford.nlp.sempre.fbalignment.lexicons.BinaryLexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.EntityLexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.UnaryLexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.WordDistance; import fig.basic.*; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.lang.reflect.Field; import java.util.*; /** @@ -23,11 +19,21 @@ public class Master { public static class Options { @Option(gloss = "Execute these commands before starting") public List scriptPaths = Lists.newArrayList(); + @Option(gloss = "Write out input lines to this path") + public String logPath; + + @Option(gloss = "Number of exchanges to keep in the context") + public int contextMaxExchanges = 1; + + @Option(gloss = "Online update weights on new examples.") + public boolean onlineLearnExamples = true; @Option(gloss = "Write out new examples to this directory") public String newExamplesPath; - @Option(gloss = "Write out input lines to this path") public String logPath; - @Option(gloss = "Online update weights on new examples.") - public boolean onlineLearnExamples; + @Option(gloss = "Write out new parameters to this directory") + public String newParamsPath; + + @Option(gloss = "Write out new grammar rules") + public String newGrammarPath; } public static Options opts = new Options(); @@ -42,13 +48,21 @@ public class Master { // Detailed information List lines = new ArrayList<>(); + public String getFormulaAnswer() { + if (ex.getPredDerivations().size() == 0) + return "(no answer)"; + else { + Derivation deriv = getDerivation(); + return deriv.getFormula() + " => " + deriv.getValue(); + } + } public String getAnswer() { if (ex.getPredDerivations().size() == 0) return "(no answer)"; else { Derivation deriv = getDerivation(); - deriv.ensureExecuted(builder.executor); - return getDerivation().getValue().toString(); + deriv.ensureExecuted(builder.executor, ex.context); + return deriv.getValue().toString(); } } public List getLines() { return lines; } @@ -79,7 +93,8 @@ public class Master { session = new Session(id); for (String path : opts.scriptPaths) processScript(session, path); - sessions.put(id, session); + if (id != null) + sessions.put(id, session); } return session; } @@ -88,16 +103,19 @@ public class Master { LogInfo.log("Enter an utterance to parse or one of the following commands:"); LogInfo.log(" (help): show this help message"); LogInfo.log(" (status): prints out status of the system"); + LogInfo.log(" (get |option|): get a command-line option (e.g., (get Parser.verbose))"); LogInfo.log(" (set |option| |value|): set a command-line option (e.g., (set Parser.verbose 5))"); LogInfo.log(" (reload): reload the grammar/parameters"); LogInfo.log(" (grammar): prints out the grammar"); - LogInfo.log(" (params): dumps all the model parameters"); + LogInfo.log(" (params [|file|]): dumps all the model parameters"); LogInfo.log(" (select |candidate index|): show information about the |index|-th candidate of the last utterance."); LogInfo.log(" (accept |candidate index|): record the |index|-th candidate as the correct answer for the last utterance."); LogInfo.log(" (answer |answer|): record |answer| as the correct answer for the last utterance (e.g., (answer (list (number 3))))."); LogInfo.log(" (rule |lhs| (|rhs_1| ... |rhs_k|) |sem|): adds a rule to the grammar (e.g., (rule $Number ($TOKEN) (NumberFn)))"); - LogInfo.log(" (execute |logical form|): adds a rule to the grammar (e.g., (execute (call + (number 3) (number 4))))"); + LogInfo.log(" (type |logical form|): perform type inference (e.g., (type (number 3)))"); + LogInfo.log(" (execute |logical form|): execute the logical form (e.g., (execute (call + (number 3) (number 4))))"); LogInfo.log(" (def |key| |value|): define a macro to replace |key| with |value| in all commands (e.g., (def type fb:type.object type)))"); + LogInfo.log(" (context [(user |user|) (date |date|) (exchange |exchange|) (graph |graph|)]): prints out or set the context"); } public void runInteractivePrompt() { @@ -119,7 +137,7 @@ public class Master { int indent = LogInfo.getIndLevel(); try { - processLine(session, line); + processQuery(session, line); } catch (Throwable t) { while (LogInfo.getIndLevel() > indent) LogInfo.end_track(); @@ -133,7 +151,7 @@ public class Master { Iterator it = LispTree.proto.parseFromFile(scriptPath); while (it.hasNext()) { LispTree tree = it.next(); - processLine(session, tree.toString()); + processQuery(session, tree.toString()); } } @@ -141,14 +159,25 @@ public class Master { // Currently, synchronize a very crude level. // In the future, refine this. // Currently need the synchronization because of writing to stdout. - public synchronized Response processLine(Session session, String line) { + public synchronized Response processQuery(Session session, String line) { + line = line.trim(); Response response = new Response(); - line = line.trim(); + // Capture log output and put it into response. + // Hack: modifying a static variable to capture the logging. + // Make sure we're synchronized! + StringWriter stringOut = new StringWriter(); + LogInfo.setFileOut(new PrintWriter(stringOut)); + if (line.startsWith("(")) handleCommand(session, line, response); else - handleQuery(session, line, response); + handleUtterance(session, line, response); + + // Clean up + for (String outLine : stringOut.toString().split("\n")) + response.lines.add(outLine); + LogInfo.setFileOut(null); // Log interaction to disk if (!Strings.isNullOrEmpty(opts.logPath)) { @@ -156,11 +185,12 @@ public class Master { out.println( Joiner.on("\t").join( Lists.newArrayList( - new Date().toString(), - session.id, - session.lastRemoteAddr != null ? session.lastRemoteAddr : "(none)", - line, - summaryString(response)))); + "date=" + new Date().toString(), + "sessionId=" + session.id, + "remote=" + session.remoteHost, + "format=" + session.format, + "query=" + line, + "response=" + summaryString(response)))); out.close(); } @@ -169,39 +199,33 @@ public class Master { String summaryString(Response response) { if (response.getExample() != null) - return response.getAnswer(); + return response.getFormulaAnswer(); if (response.getLines().size() > 0) return response.getLines().get(0); return null; } - private void handleQuery(Session session, String query, Response response) { - session.lastQuery = query; + private void handleUtterance(Session session, String query, Response response) { + session.updateContext(); + // Create example Example.Builder b = new Example.Builder(); - b.setId("session:" + session.id); - int slashIndex = query.indexOf('/'); - if (slashIndex != -1) { - // if query = "where was obama born? /place_of_birth", that constrains - // derivations to only ones whose formula contains place_of_birth. - b.setDerivConstraint(new DerivationConstraint(query.substring(slashIndex + 1))); - query = query.substring(0, slashIndex).trim(); - } b.setUtterance(query); - + b.setContext(session.context); Example ex = b.createExample(); - ex.preprocess(); + + ex.preprocess(LanguageAnalyzer.getSingleton()); // Parse! - builder.parser.parse(builder.params, ex); + builder.parser.parse(builder.params, ex, false); - session.examples.add(ex); response.ex = ex; if (ex.predDerivations.size() > 0) { response.candidateIndex = 0; printDerivation(response.getDerivation()); } + session.updateContext(ex, opts.contextMaxExchanges); } private void printDerivation(Derivation deriv) { @@ -224,23 +248,8 @@ public class Master { } private void handleCommand(Session session, String line, Response response) { - // Capture log output and put it into response. - // Hack: modifying a static variable to capture the logging. - // Make sure we're synchronized! - StringWriter stringOut = new StringWriter(); - LogInfo.setFileOut(new PrintWriter(stringOut)); - - handleCommandHelper(session, line, response); - - for (String outLine : stringOut.toString().split("\n")) - response.lines.add(outLine); - - LogInfo.setFileOut(null); - } - - private void handleCommandHelper(Session session, String line, Response response) { LispTree tree = LispTree.proto.parseFromString(line); - tree = Grammar.applyMacros(session.macros, tree); + tree = builder.grammar.applyMacros(tree); String command = tree.child(0).value; @@ -258,29 +267,42 @@ public class Master { for (Rule rule : builder.grammar.rules) LogInfo.logs("%s", rule.toLispTree()); } else if (command.equals("params")) { - builder.params.write(LogInfo.stdout); - } else if (command.equals("set")) { - if (tree.children.size() != 3) { - LogInfo.log("Invalid usage: (set |key| |value|)"); + if (tree.children.size() == 1) { + builder.params.write(LogInfo.stdout); + if (LogInfo.getFileOut() != null) + builder.params.write(LogInfo.getFileOut()); + } else { + builder.params.write(tree.child(1).value); + } + } else if (command.equals("get")) { + if (tree.children.size() != 2) { + LogInfo.log("Invalid usage: (get |option|)"); return; } - String key = tree.child(1).value; + String option = tree.child(1).value; + LogInfo.logs("%s", getOptionsParser().getValue(option)); + } else if (command.equals("set")) { + if (tree.children.size() != 3) { + LogInfo.log("Invalid usage: (set |option| |value|)"); + return; + } + String option = tree.child(1).value; String value = tree.child(2).value; - if (!getOptionsParser().parse(new String[]{"-" + key, value})) - LogInfo.log("Unknown option: " + key); - } else if (command.equals("select") || command.equals("accept")) { + if (!getOptionsParser().parse(new String[] {"-" + option, value})) + LogInfo.log("Unknown option: " + option); + } else if (command.equals("select") || command.equals("accept") || + command.equals("s") || command.equals("a")) { // Select an answer if (tree.children.size() != 2) { LogInfo.logs("Invalid usage: (%s |candidate index|)", command); return; } - if (session.examples.size() == 0) { + Example ex = session.getLastExample(); + if (ex == null) { LogInfo.log("No examples - please enter a query first."); return; } - - Example ex = session.examples.get(session.examples.size() - 1); int index = Integer.parseInt(tree.child(1).value); if (index < 0 || index >= ex.predDerivations.size()) { LogInfo.log("Candidate index out of range: " + index); @@ -289,13 +311,22 @@ public class Master { response.ex = ex; response.candidateIndex = index; - + session.updateContextWithNewAnswer(ex, response.getDerivation()); printDerivation(response.getDerivation()); - // Set logical form - if (command.equals("accept")) { + // Add a training example. While the user selects a particular derivation, there are three ways to interpret this signal: + // 1. This is the correct derivation (Derivation). + // 2. This is the correct logical form (Formula). + // 3. This is the correct denotation (Value). + // Currently: + // - Parameters based on the denotation. + // - Grammar rules are induced based on the denotation. + // We always save the logical form and the denotation (but not the entire + // derivation) in the example. + if (command.equals("accept") || command.equals("a")) { ex.setTargetFormula(response.getDerivation().getFormula()); ex.setTargetValue(response.getDerivation().getValue()); + ex.setContext(session.getContextExcludingLast()); addNewExample(ex); } } else if (command.equals("answer")) { @@ -303,13 +334,12 @@ public class Master { LogInfo.log("Missing answer."); } - if (session.examples.size() == 0) { - LogInfo.log("No examples - please enter a query first."); + // Set the target value. + Example ex = session.getLastExample(); + if (ex == null) { + LogInfo.log("Please enter a query first."); return; } - - // Set the target value. - Example ex = session.examples.get(session.examples.size() - 1); ex.setTargetValue(Values.fromLispTree(tree.child(1))); addNewExample(ex); } else if (command.equals("rule")) { @@ -320,15 +350,21 @@ public class Master { // Need to update the parser given that the grammar has changed. builder.parser = null; builder.buildUnspecified(); + } else if (command.equals("type")) { + LogInfo.logs("%s", TypeInference.inferType(Formulas.fromLispTree(tree.child(1)))); } else if (command.equals("execute")) { - Executor.Response execResponse = builder.executor.execute(Formulas.fromLispTree(tree.child(1))); + Example ex = session.getLastExample(); + ContextValue context = (ex != null ? ex.context : session.context); + Executor.Response execResponse = builder.executor.execute(Formulas.fromLispTree(tree.child(1)), context); LogInfo.logs("%s", execResponse.value); } else if (command.equals("def")) { - if (tree.children.size() != 3 || !tree.child(1).isLeaf()) { - LogInfo.log("Invalid usage: (def |name| |value|)"); - return; + builder.grammar.interpretMacroDef(tree); + } else if (command.equals("context")) { + if (tree.children.size() == 1) { + LogInfo.logs("%s", session.context); + } else { + session.context = new ContextValue(tree); } - session.macros.put(tree.child(1).value, tree.child(2)); } else { LogInfo.log("Invalid command: " + tree); } @@ -339,54 +375,69 @@ public class Master { Example ex = new Example.Builder() .setId(origEx.id) .setUtterance(origEx.utterance) + .setContext(origEx.context) .setTargetFormula(origEx.targetFormula) .setTargetValue(origEx.targetValue) .createExample(); if (!Strings.isNullOrEmpty(opts.newExamplesPath)) { - LogInfo.log("Added new example."); - PrintWriter out = IOUtils.openOutAppendHard(opts.newExamplesPath); - out.println(ex.toJson()); - out.close(); + LogInfo.begin_track("Adding new example"); + Dataset.appendExampleToFile(opts.newExamplesPath, ex); + LogInfo.end_track(); } if (opts.onlineLearnExamples) { - LogInfo.log("Updating parameters."); + LogInfo.begin_track("Updating parameters"); learner.onlineLearnExample(origEx); + if (!Strings.isNullOrEmpty(opts.newParamsPath)) + builder.params.write(opts.newParamsPath); + LogInfo.end_track(); } } public static OptionsParser getOptionsParser() { OptionsParser parser = new OptionsParser(); - parser.registerAll( - new Object[]{ - "Master", Master.opts, - "Builder", Builder.opts, - "Grammar", Grammar.opts, - "Derivation", Derivation.opts, - "Parser", Parser.opts, - "BeamParser", BeamParser.opts, - "SparqlExecutor", SparqlExecutor.opts, - "Dataset", Dataset.opts, - "Params", Params.opts, - "Learner", Learner.opts, - "SemanticFn", SemanticFn.opts, - "LexiconFn", LexiconFn.opts, - "SelectFn", SelectFn.opts, - "MergeFn", MergeFn.opts, - "JoinFn", JoinFn.opts, - "DescriptionValue", DescriptionValue.opts, - "FeatureExtractor", FeatureExtractor.opts, - "LanguageInfo", LanguageInfo.opts, - "EntityLexicon", EntityLexicon.opts, - "BinaryLexicon", BinaryLexicon.opts, - "UnaryLexicon", UnaryLexicon.opts, - "FreebaseInfo", FreebaseInfo.opts, - "BridgeFn", BridgeFn.opts, - "WordDistance", WordDistance.opts, - "FormulaRetriever",FormulaRetriever.opts, - "Lexicon", Lexicon.opts, - }); + // Dynamically figure out which options we need to load + // To specify this: + // java -Dmodules=core,freebase + List modules = Arrays.asList(System.getProperty("modules", "core").split(",")); + + // All options are assumed to be of the form opts. + // Read the module-classes.txt file, which specifies which classes are + // associated with each module. + List args = new ArrayList(); + for (String line : IOUtils.readLinesHard("module-classes.txt")) { + + // Example: core edu.stanford.nlp.sempre.Grammar + String[] tokens = line.split(" "); + if (tokens.length != 2) throw new RuntimeException("Invalid: " + line); + String module = tokens[0]; + String className = tokens[1]; + if (!modules.contains(tokens[0])) continue; + + // Group (e.g., Grammar) + String[] classNameTokens = className.split("\\."); + String group = classNameTokens[classNameTokens.length - 1]; + + // Object (e.g., Grammar.opts) + Object opts = null; + try { + for (Field field : Class.forName(className).getDeclaredFields()) { + if (!"opts".equals(field.getName())) continue; + opts = field.get(null); + } + } catch (Throwable t) { + System.out.println("Problem processing: " + line); + throw new RuntimeException(t); + } + + if (opts != null) { + args.add(group); + args.add(opts); + } + } + + parser.registerAll(args.toArray(new Object[0])); return parser; } } diff --git a/src/edu/stanford/nlp/sempre/MergeFn.java b/src/edu/stanford/nlp/sempre/MergeFn.java index b4a4c6e..51d0550 100644 --- a/src/edu/stanford/nlp/sempre/MergeFn.java +++ b/src/edu/stanford/nlp/sempre/MergeFn.java @@ -2,9 +2,7 @@ package edu.stanford.nlp.sempre; import fig.basic.LispTree; import fig.basic.Option; - -import java.util.Collections; -import java.util.List; +import fig.basic.LogInfo; /** * Takes two unaries and merges (takes the intersection) of them. @@ -15,16 +13,13 @@ public class MergeFn extends SemanticFn { public static class Options { @Option(gloss = "whether to do a hard type-check") public boolean hardTypeCheck = true; + + @Option public boolean showTypeCheckFailures = false; @Option(gloss = "Verbose") public int verbose = 0; } - private FbFormulasInfo fbFormulaInfo = null; public static Options opts = new Options(); - public MergeFn() { - fbFormulaInfo = FbFormulasInfo.getSingleton(); - } - MergeFormula.Mode mode; // How to merge Formula formula; // Optional: merge with this if exists @@ -36,57 +31,52 @@ public class MergeFn extends SemanticFn { } } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MergeFn mergeFn = (MergeFn) o; - if (formula != null ? !formula.equals(mergeFn.formula) : mergeFn.formula != null) return false; - if (mode != mergeFn.mode) return false; - return true; - } + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + Formula result; + if (c.getChildren().size() == 1) + result = c.child(0).formula; + else if (c.getChildren().size() == 2) + result = new MergeFormula(mode, c.child(0).formula, c.child(1).formula); + else + throw new RuntimeException("Bad args: " + c.getChildren()); - public List call(Example ex, Callable c) { - Formula result; - if (c.getChildren().size() == 1) - result = c.child(0).formula; - else if (c.getChildren().size() == 2) - result = new MergeFormula(mode, c.child(0).formula, c.child(1).formula); - else - throw new RuntimeException("Bad args: " + c.getChildren()); + // Compute resulting type + Derivation child0 = c.child(0); + Derivation child1 = c.child(1); + SemType type = child0.type.meet(child1.type); + FeatureVector features = new FeatureVector(); + if (opts.verbose >= 5) + LogInfo.logs("MergeFn: %s | %s | %s", child0, child1, type); - // Compute resulting type - Derivation child0 = c.child(0); - Derivation child1 = c.child(1); - SemType type = child0.type.meet(child1.type); - FeatureVector features = new FeatureVector(); + if (!type.isValid()) { + if (opts.hardTypeCheck) { + if (opts.showTypeCheckFailures) + LogInfo.warnings("MergeFn: type check failed: [%s : %s] AND [%s : %s]", child0.formula, child0.type, child1.formula, child1.type); + return null; + } + } - if (!type.isValid()) { - if (opts.hardTypeCheck) - return Collections.emptyList(); // Don't accept logical forms that don't type check - else { - if (FeatureExtractor.containsDomain("typeCheck")) - features.add("typeCheck", "mergeMismatch"); // Just add a feature + if (formula != null) + result = new MergeFormula(mode, formula, result); + + Derivation deriv = new Derivation.Builder() + .withCallable(c) + .formula(result) + .type(type) + .localFeatureVector(features) + .createDerivation(); + + if (SemanticFn.opts.trackLocalChoices) { + deriv.addLocalChoice( + "MergeFn " + + child0.startEndString(ex.getTokens()) + " " + child0.formula + " AND " + + child1.startEndString(ex.getTokens()) + " " + child1.formula); + } + return deriv; } - } - - if (formula != null) - result = new MergeFormula(mode, formula, result); - - Derivation deriv = new Derivation.Builder() - .withCallable(c) - .formula(result) - .type(type) - .localFeatureVector(features) - .createDerivation(); - - if (SemanticFn.opts.trackLocalChoices) { - deriv.localChoices.add( - "MergeFn " + - child0.startEndString(ex.getTokens()) + " " + child0.formula + " AND " + - child1.startEndString(ex.getTokens()) + " " + child1.formula); - } - - return Collections.singletonList(deriv); + }; } } diff --git a/src/edu/stanford/nlp/sempre/MergeFormula.java b/src/edu/stanford/nlp/sempre/MergeFormula.java index 9de6f1f..7153ef9 100644 --- a/src/edu/stanford/nlp/sempre/MergeFormula.java +++ b/src/edu/stanford/nlp/sempre/MergeFormula.java @@ -3,6 +3,8 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * Takes two unary formulas and performs either the intersection or union. * @@ -33,6 +35,16 @@ public class MergeFormula extends Formula { return result == null ? new MergeFormula(mode, child1.map(func), child2.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) { + res.addAll(child1.mapToList(func, alwaysRecurse)); + res.addAll(child2.mapToList(func, alwaysRecurse)); + } + return res; + } + public static Mode parseMode(String mode) { if ("and".equals(mode)) return Mode.and; if ("or".equals(mode)) return Mode.or; @@ -48,7 +60,7 @@ public class MergeFormula extends Formula { if (!this.child2.equals(that.child2)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + mode.toString().hashCode(); // Note: don't call hashCode() on mode directly. diff --git a/src/edu/stanford/nlp/sempre/MultipleDerivationStream.java b/src/edu/stanford/nlp/sempre/MultipleDerivationStream.java new file mode 100644 index 0000000..a8d155b --- /dev/null +++ b/src/edu/stanford/nlp/sempre/MultipleDerivationStream.java @@ -0,0 +1,48 @@ +package edu.stanford.nlp.sempre; + +// Encapsulates the production of possibly many Derivations. +// The subclass has to maintain the cursor to keep track of which is coming next. +public abstract class MultipleDerivationStream implements DerivationStream { + private Derivation nextDeriv; // Next one to return. + int numGenerated = 0; + + // Override this class: should create a new Derivation. + // Return null if there are no more. + public abstract Derivation createDerivation(); + + @Override public boolean hasNext() { + if (nextDeriv != null) return true; // Still one in the queue + nextDeriv = createDerivation(); // Ask for another + return nextDeriv != null; + } + + @Override + public Derivation next() { + if (nextDeriv == null) throw new RuntimeException("No more derivations!"); + Derivation deriv = nextDeriv; + if (FeatureExtractor.containsDomain("derivRank")) { + numGenerated++; + if (numGenerated <= 3) + deriv.addFeature("derivRank", deriv.rule.sem.toString() + " " + numGenerated); + else if (numGenerated <= 5) + deriv.addFeature("derivRank", deriv.rule.sem.toString() + " 4:5"); + else if (numGenerated <= 10) + deriv.addFeature("derivRank", deriv.rule.sem.toString() + " 6:10"); + else + deriv.addFeature("derivRank", deriv.rule.sem.toString() + " 11:"); + } + nextDeriv = createDerivation(); + return deriv; + } + + @Override + public Derivation peek() { + if (nextDeriv == null) throw new RuntimeException("No more derivations!"); + return nextDeriv; + } + + @Override public void remove() { throw new RuntimeException("Cannot remove from DerivationStream"); } + + // Default: but can overload this if desired + @Override public int estimatedSize() { return 2; } +} diff --git a/src/edu/stanford/nlp/sempre/NaiveKnowledgeGraph.java b/src/edu/stanford/nlp/sempre/NaiveKnowledgeGraph.java new file mode 100644 index 0000000..3bef98a --- /dev/null +++ b/src/edu/stanford/nlp/sempre/NaiveKnowledgeGraph.java @@ -0,0 +1,192 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; +import fig.basic.*; + +/** + * Represent a knowledge graph explicitly as triples (e1, r, e2). + * + * The graph is immutable. Once the graph is initialized, we precompute several mappings + * (e.g., list of all outgoing edges from each entity e). + * + * @author ppasupat + */ +public class NaiveKnowledgeGraph extends KnowledgeGraph { + + // Represent a triple (entity, relation, entity) + public static class KnowledgeGraphTriple { + public final Value e1, r, e2; + + public KnowledgeGraphTriple(Value e1, Value r, Value e2) { + this.e1 = e1; + this.r = r; + this.e2 = e2; + } + + public KnowledgeGraphTriple(String e1, String r, String e2) { + this.e1 = new StringValue(e1); + this.r = new StringValue(r); + this.e2 = new StringValue(e2); + } + + public KnowledgeGraphTriple(LispTree tree) { + if (tree.children.size() != 3) + throw new RuntimeException("Invalid triple size (" + tree.children.size() + " != 3): " + + tree); + this.e1 = valueFromLispTree(tree.child(0)); + this.r = valueFromLispTree(tree.child(1)); + this.e2 = valueFromLispTree(tree.child(2)); + } + + protected static Value valueFromLispTree(LispTree tree) { + if (tree.isLeaf()) return new NameValue(tree.value, null); + return Values.fromLispTree(tree); + } + + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild(e1.toLispTree()); + tree.addChild(r.toLispTree()); + tree.addChild(e2.toLispTree()); + return tree; + } + } + + // Simplest graph representation: triples of values + public final List triples; + + // ============================================================ + // Constructor / Precomputation + // ============================================================ + + public Map> relationToTriples; + public Map> firstToTriples; + public Map> secondToTriples; + + public NaiveKnowledgeGraph(Collection triples) { + this.triples = new ArrayList<>(triples); + precomputeMappings(); + } + + public void precomputeMappings() { + relationToTriples = new HashMap<>(); + firstToTriples = new HashMap<>(); + secondToTriples = new HashMap<>(); + for (KnowledgeGraphTriple triple : triples) { + MapUtils.addToList(relationToTriples, triple.r, triple); + MapUtils.addToList(firstToTriples, triple.e1, triple); + MapUtils.addToList(secondToTriples, triple.e2, triple); + } + } + + @Override + public Collection getFuzzyMatchedFormulas(String term, FuzzyMatchFn.FuzzyMatchFnMode mode) { + throw new RuntimeException("Not implemented yet"); + } + + @Override + public Collection getAllFormulas(FuzzyMatchFn.FuzzyMatchFnMode mode) { + throw new RuntimeException("Not implemented yet"); + } + + // ============================================================ + // Queries + // ============================================================ + + @Override + public List joinFirst(Value r, Collection firsts) { + Value reversed = isReversedRelation(r); + if (reversed != null) + return joinSecond(reversed, firsts); + List seconds = new ArrayList<>(); + List relationTriples = relationToTriples.get(r); + if (relationTriples != null) { + for (KnowledgeGraphTriple triple : relationTriples) { + if (firsts.contains(triple.e1)) + seconds.add(triple.e2); + } + } + return seconds; + } + + @Override + public List joinSecond(Value r, Collection seconds) { + Value reversed = isReversedRelation(r); + if (reversed != null) + return joinFirst(reversed, seconds); + List firsts = new ArrayList<>(); + List relationTriples = relationToTriples.get(r); + if (relationTriples != null) { + for (KnowledgeGraphTriple triple : relationTriples) { + if (seconds.contains(triple.e2)) + firsts.add(triple.e1); + } + } + return firsts; + } + + @Override + public List> filterFirst(Value r, Collection firsts) { + Value reversed = isReversedRelation(r); + if (reversed != null) + return getReversedPairs(filterSecond(reversed, firsts)); + List> pairs = new ArrayList<>(); + List relationTriples = relationToTriples.get(r); + if (relationTriples != null) { + for (KnowledgeGraphTriple triple : relationTriples) { + if (firsts.contains(triple.e1)) + pairs.add(new Pair<>(triple.e1, triple.e2)); + } + } + return pairs; + } + + @Override + public List> filterSecond(Value r, Collection seconds) { + Value reversed = isReversedRelation(r); + if (reversed != null) + return getReversedPairs(filterFirst(reversed, seconds)); + List> pairs = new ArrayList<>(); + List relationTriples = relationToTriples.get(r); + if (relationTriples != null) { + for (KnowledgeGraphTriple triple : relationTriples) { + if (seconds.contains(triple.e2)) + pairs.add(new Pair<>(triple.e1, triple.e2)); + } + } + return pairs; + } + + // ============================================================ + // LispTree conversion + // ============================================================ + + /** + * Convert LispTree to KnowledgeGraph + * + * The |tree| should look like + * + * (graph NaiveKnowledgeGraph + * ((string Obama) (string "born in") (string Hawaii)) + * ((string Einstein) (string "born in") (string Ulm)) + * ...) + */ + public static KnowledgeGraph fromLispTree(LispTree tree) { + List triples = new ArrayList<>(); + for (int i = 2; i < tree.children.size(); i++) { + triples.add(new KnowledgeGraphTriple(tree.child(i))); + } + return new NaiveKnowledgeGraph(triples); + } + + @Override + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild("graph"); + tree.addChild("NaiveKnowledgeGraph"); + for (KnowledgeGraphTriple triple : triples) { + tree.addChild(triple.toLispTree()); + } + return tree; + } +} diff --git a/src/edu/stanford/nlp/sempre/NameValue.java b/src/edu/stanford/nlp/sempre/NameValue.java index 332fd27..c0c841b 100644 --- a/src/edu/stanford/nlp/sempre/NameValue.java +++ b/src/edu/stanford/nlp/sempre/NameValue.java @@ -42,9 +42,11 @@ public class NameValue extends Value { } @Override public int hashCode() { return id.hashCode(); } - @Override public boolean equals(Object thatObj) { - if (!(thatObj instanceof NameValue)) return false; - NameValue that = (NameValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NameValue that = (NameValue) o; + // Note: only check id, not description return this.id.equals(that.id); } } diff --git a/src/edu/stanford/nlp/sempre/NotFormula.java b/src/edu/stanford/nlp/sempre/NotFormula.java index 4430ac3..80a715f 100644 --- a/src/edu/stanford/nlp/sempre/NotFormula.java +++ b/src/edu/stanford/nlp/sempre/NotFormula.java @@ -3,6 +3,8 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * (not expression) returns the truth value which is opposite of expression. * @@ -25,6 +27,14 @@ public class NotFormula extends Formula { return result == null ? new NotFormula(child.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) + res.addAll(child.mapToList(func, alwaysRecurse)); + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof NotFormula)) return false; @@ -32,7 +42,7 @@ public class NotFormula extends Formula { if (!this.child.equals(that.child)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + child.hashCode(); diff --git a/src/edu/stanford/nlp/sempre/NullExecutor.java b/src/edu/stanford/nlp/sempre/NullExecutor.java index 665d8ae..63ac6aa 100644 --- a/src/edu/stanford/nlp/sempre/NullExecutor.java +++ b/src/edu/stanford/nlp/sempre/NullExecutor.java @@ -6,7 +6,7 @@ package edu.stanford.nlp.sempre; * @author Percy Liang */ public class NullExecutor extends Executor { - public Response execute(Formula formula) { + public Response execute(Formula formula, ContextValue context) { return new Response(null); } } diff --git a/src/edu/stanford/nlp/sempre/NullTypeLookup.java b/src/edu/stanford/nlp/sempre/NullTypeLookup.java new file mode 100644 index 0000000..41d151c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/NullTypeLookup.java @@ -0,0 +1,17 @@ +package edu.stanford.nlp.sempre; + +/** + * Default implementation of TypeLookup: just return null (I don't know what + * the type is). + */ +public class NullTypeLookup implements TypeLookup { + @Override + public SemType getEntityType(String entity) { + return null; + } + + @Override + public SemType getPropertyType(String property) { + return null; + } +} diff --git a/src/edu/stanford/nlp/sempre/NumberFn.java b/src/edu/stanford/nlp/sempre/NumberFn.java index 741f4e7..032656a 100644 --- a/src/edu/stanford/nlp/sempre/NumberFn.java +++ b/src/edu/stanford/nlp/sempre/NumberFn.java @@ -1,7 +1,8 @@ package edu.stanford.nlp.sempre; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; +import fig.basic.*; /** * Maps a string to a number (double). @@ -9,25 +10,99 @@ import java.util.List; * @author Percy Liang */ public class NumberFn extends SemanticFn { - // TODO: handle measurements too (e.g., 3cm) + private List requests; // List of types of fields to get (e.g., NUMBER) - @Override - public boolean equals(Object o) { - if (this == o) return true; - return o != null && getClass() == o.getClass(); + private boolean request(String req) { + return requests == null || requests.contains(req); } - public List call(Example ex, Callable c) { - String value = ex.languageInfo.getNormalizedNerSpan("NUMBER", c.getStart(), c.getEnd()); - if (value == null) return Collections.emptyList(); + public void init(LispTree tree) { + super.init(tree); + if (tree.children.size() > 1) { + requests = new ArrayList(); + for (int i = 1; i < tree.children.size(); i++) + requests.add(tree.child(1).value); + } + } - NumberValue numberValue = new NumberValue(Double.parseDouble(value)); + // TODO(pliang): handle measurements too (e.g., 3cm) + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + public Derivation createDerivation() { + // Numbers: If it is an integer, set its type to integer. Otherwise, use float. + if (request("NUMBER")) { + String value = ex.languageInfo.getNormalizedNerSpan("NUMBER", c.getStart(), c.getEnd()); + if (value != null) { + try { + NumberValue numberValue = new NumberValue(Double.parseDouble(value)); + SemType type = numberValue.value == (int) numberValue.value ? SemType.intType : SemType.floatType; + return new Derivation.Builder() + .withCallable(c) + .formula(new ValueFormula<>(numberValue)) + .type(type) + .createDerivation(); + } catch (NumberFormatException e) { + LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value); + } + } + } - return Collections.singletonList( - new Derivation.Builder() - .withCallable(c) - .formula(new ValueFormula(numberValue)) - .type(SemType.numberType) - .createDerivation()); + // Ordinals + if (request("ORDINAL")) { + String value = ex.languageInfo.getNormalizedNerSpan("ORDINAL", c.getStart(), c.getEnd()); + if (value != null) { + try { + NumberValue numberValue = new NumberValue(Double.parseDouble(value), "fb:en.ordinal_number"); + SemType type = SemType.intType; + return new Derivation.Builder() + .withCallable(c) + .formula(new ValueFormula<>(numberValue)) + .type(type) + .createDerivation(); + } catch (NumberFormatException e) { + LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value); + } + } + } + + // Percents + if (request("PERCENT")) { + String value = ex.languageInfo.getNormalizedNerSpan("PERCENT", c.getStart(), c.getEnd()); + if (value != null) { + try { + NumberValue numberValue = new NumberValue(0.01 * Double.parseDouble(value.substring(1))); + SemType type = SemType.floatType; + return new Derivation.Builder() + .withCallable(c) + .formula(new ValueFormula<>(numberValue)) + .type(type) + .createDerivation(); + } catch (NumberFormatException e) { + LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value); + } + } + } + + // Money + if (request("MONEY")) { + String value = ex.languageInfo.getNormalizedNerSpan("MONEY", c.getStart(), c.getEnd()); + if (value != null) { + try { + NumberValue numberValue = new NumberValue(Double.parseDouble(value.substring(1)), "fb:en.dollar"); + SemType type = SemType.floatType; + return new Derivation.Builder() + .withCallable(c) + .formula(new ValueFormula<>(numberValue)) + .type(type) + .createDerivation(); + } catch (NumberFormatException e) { + LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value); + } + } + } + + return null; + } + }; } } diff --git a/src/edu/stanford/nlp/sempre/NumberValue.java b/src/edu/stanford/nlp/sempre/NumberValue.java index 3b1df56..740fee7 100644 --- a/src/edu/stanford/nlp/sempre/NumberValue.java +++ b/src/edu/stanford/nlp/sempre/NumberValue.java @@ -11,6 +11,7 @@ import fig.basic.Fmt; */ public class NumberValue extends Value { public static final String unitless = "fb:en.unitless"; + public static final String yearUnit = "fb:en.year"; public final double value; public final String unit; // What measurement (e.g., "fb:en.meter" or unitless) @@ -39,9 +40,10 @@ public class NumberValue extends Value { } @Override public int hashCode() { return Double.valueOf(value).hashCode(); } - @Override public boolean equals(Object thatObj) { - if (!(thatObj instanceof NumberValue)) return false; - NumberValue that = (NumberValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NumberValue that = (NumberValue) o; if (this.value != that.value) return false; // Warning: doing exact equality checking if (!this.unit.equals(that.unit)) return false; return true; diff --git a/src/edu/stanford/nlp/sempre/Params.java b/src/edu/stanford/nlp/sempre/Params.java index 37b7611..e756051 100644 --- a/src/edu/stanford/nlp/sempre/Params.java +++ b/src/edu/stanford/nlp/sempre/Params.java @@ -31,7 +31,7 @@ public class Params { @Option(gloss = "Use the AdaGrad algorithm (different step size for each coordinate)") public boolean adaptiveStepSize = true; @Option(gloss = "Use dual averaging") public boolean dualAveraging = false; - @Option(gloss = "Whether to do lazy l1 reg updates") public String l1Reg="none"; + @Option(gloss = "Whether to do lazy l1 reg updates") public String l1Reg = "none"; @Option(gloss = "L1 reg coefficient") public double l1RegCoeff = 0d; } public static Options opts = new Options(); @@ -41,29 +41,35 @@ public class Params { NONE; } private L1Reg parseReg(String l1Reg) { - if("lazy".equals(l1Reg)) return L1Reg.LAZY; - if("nonlazy".equals(l1Reg)) return L1Reg.NONLAZY; - if("none".equals(l1Reg)) return L1Reg.NONE; + if ("lazy".equals(l1Reg)) return L1Reg.LAZY; + if ("nonlazy".equals(l1Reg)) return L1Reg.NONLAZY; + if ("none".equals(l1Reg)) return L1Reg.NONE; throw new RuntimeException("not legal l1reg"); } private L1Reg l1Reg = parseReg(opts.l1Reg); // Discriminative weights - private HashMap weights = new HashMap(); + private HashMap weights = new HashMap<>(); // For AdaGrad - Map sumSquaredGradients = new HashMap(); + Map sumSquaredGradients = new HashMap<>(); // For dual averaging - Map sumGradients = new HashMap(); + Map sumGradients = new HashMap<>(); // Number of stochastic updates we've made so far (for determining step size). int numUpdates; - //for lazy l1-reg update - Map l1UpdateTimeMap = new HashMap(); - + // for lazy l1-reg update + Map l1UpdateTimeMap = new HashMap<>(); + // Initialize the weights + public void init(List> initialization) { + if (!weights.isEmpty()) + throw new RuntimeException("Initialization is not legal when there are non-zero weights"); + for (Pair pair: initialization) + weights.put(pair.getFirst(), pair.getSecond()); + } // Read parameters from |path|. public void read(String path) { @@ -83,6 +89,25 @@ public class Params { LogInfo.end_track(); } + // Read parameters from |path|. + public void read(String path, String prefix) { + LogInfo.begin_track("Reading parameters from %s", path); + try { + BufferedReader in = IOUtils.openIn(path); + String line; + while ((line = in.readLine()) != null) { + String[] pair = Lists.newArrayList(Splitter.on('\t').split(line)).toArray(new String[2]); + weights.put(pair[0], Double.parseDouble(pair[1])); + weights.put(prefix + pair[0], Double.parseDouble(pair[1])); + } + in.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.logs("Read %s weights", weights.size()); + LogInfo.end_track(); + } + // Update weights by adding |gradient| (modified appropriately with step size). public synchronized void update(Map gradient) { numUpdates++; @@ -90,33 +115,32 @@ public class Params { for (Map.Entry entry : gradient.entrySet()) { String f = entry.getKey(); double g = entry.getValue(); - if (g == 0) continue; + if (g * g == 0) continue; // In order to not divide by zero double stepSize = computeStepSize(f, g); if (opts.dualAveraging) { if (!opts.adaptiveStepSize && opts.stepSizeReduction != 0) throw new RuntimeException("Dual averaging not supported when " + - "step-size changes across iterations for " + - "features for which the gradient is zero"); + "step-size changes across iterations for " + + "features for which the gradient is zero"); MapUtils.incr(sumGradients, f, g); MapUtils.set(weights, f, stepSize * sumGradients.get(f)); - } - else { - if(stepSize*g ==Double.POSITIVE_INFINITY || stepSize*g == Double.NEGATIVE_INFINITY) { - LogInfo.logs("WEIRD FEATURE UPDATE: feature=%s, currentWeight=%s, stepSize=%s, gradient=%s",f,getWeight(f),stepSize,g); + } else { + if (stepSize * g == Double.POSITIVE_INFINITY || stepSize * g == Double.NEGATIVE_INFINITY) { + LogInfo.logs("WEIRD FEATURE UPDATE: feature=%s, currentWeight=%s, stepSize=%s, gradient=%s", f, getWeight(f), stepSize, g); throw new RuntimeException("Gradient absolute value is too large or too small"); } MapUtils.incr(weights, f, stepSize * g); } } - //non lazy implementation goes over all weights - if(l1Reg==L1Reg.NONLAZY) { + // non lazy implementation goes over all weights + if (l1Reg == L1Reg.NONLAZY) { Set features = new HashSet(weights.keySet()); for (String f :features) { - double stepSize = computeStepSize(f, 0d); //no update for gradient here - double update = opts.l1RegCoeff* -Math.signum(MapUtils.getDouble(weights, f, opts.defaultWeight)); - clipUpdate(f, stepSize*update); + double stepSize = computeStepSize(f, 0d); // no update for gradient here + double update = opts.l1RegCoeff * -Math.signum(MapUtils.getDouble(weights, f, opts.defaultWeight)); + clipUpdate(f, stepSize * update); } } } @@ -124,9 +148,9 @@ public class Params { private double computeStepSize(String feature, double gradient) { if (opts.adaptiveStepSize) { MapUtils.incr(sumSquaredGradients, feature, gradient * gradient); - //ugly - adding one to the denominator when using l1 reg. - if(l1Reg!=L1Reg.NONE) - return opts.initStepSize / (Math.sqrt(sumSquaredGradients.get(feature)+1)); + // ugly - adding one to the denominator when using l1 reg. + if (l1Reg != L1Reg.NONE) + return opts.initStepSize / (Math.sqrt(sumSquaredGradients.get(feature) + 1)); else return opts.initStepSize / Math.sqrt(sumSquaredGradients.get(feature)); } else { @@ -139,36 +163,35 @@ public class Params { */ private void clipUpdate(String f, double update) { double currWeight = MapUtils.getDouble(weights, f, 0); - if(currWeight == 0) + if (currWeight == 0) return; - if(currWeight*(currWeight+update)<0.0) { + if (currWeight * (currWeight + update) < 0.0) { weights.remove(f); - } - else { + } else { MapUtils.incr(weights, f, update); } } private void lazyL1Update(String f) { - if(sumSquaredGradients.get(f)==null) { + if (sumSquaredGradients.get(f) == null || l1UpdateTimeMap.get(f) == null) { l1UpdateTimeMap.put(f, numUpdates); return; } - int numOfIter = numUpdates-l1UpdateTimeMap.get(f); - if(numOfIter<=0) { + int numOfIter = numUpdates - l1UpdateTimeMap.get(f); + if (numOfIter <= 0) { l1UpdateTimeMap.put(f, numUpdates); return; } - double stepSize = (numOfIter * opts.initStepSize) / (Math.sqrt(sumSquaredGradients.get(f)+1)); + double stepSize = (numOfIter * opts.initStepSize) / (Math.sqrt(sumSquaredGradients.get(f) + 1)); double update = -opts.l1RegCoeff * Math.signum(MapUtils.getDouble(weights, f, 0.0)); - clipUpdate(f, stepSize*update); + clipUpdate(f, stepSize * update); l1UpdateTimeMap.put(f, numUpdates); } public synchronized double getWeight(String f) { - if(l1Reg==L1Reg.LAZY) + if (l1Reg == L1Reg.LAZY) lazyL1Update(f); if (opts.initWeightsRandomly) return MapUtils.getDouble(weights, f, 2 * opts.initRandom.nextDouble() - 1); @@ -210,10 +233,43 @@ public class Params { } public synchronized void finalizeWeights() { - if(l1Reg==L1Reg.LAZY) { - Set features = new HashSet(weights.keySet()); - for(String f: features) + if (l1Reg == L1Reg.LAZY) { + Set features = new HashSet<>(weights.keySet()); + for (String f : features) lazyL1Update(f); } } + + public Params copyParams() { + Params result = new Params(); + for (String feature : this.getWeights().keySet()) { + result.weights.put(feature, this.getWeight(feature)); + } + return result; + } + + // copy params starting with prefix and drop the prefix + public Params copyParamsByPrefix(String prefix) { + Params result = new Params(); + for (String feature : this.getWeights().keySet()) { + if (feature.startsWith(prefix)) { + String newFeature = feature.substring(prefix.length()); + result.weights.put(newFeature, this.getWeight(feature)); + } + } + return result; + } + + public boolean isEmpty() { + return weights.size() == 0; + } + + public Params getRandomWeightParams() { + Random rand = new Random(); + Params result = new Params(); + for (String feature : this.getWeights().keySet()) { + result.weights.put(feature, 2 * rand.nextDouble() - 1); // between -1 and 1 + } + return result; + } } diff --git a/src/edu/stanford/nlp/sempre/ParaphraseModel.java b/src/edu/stanford/nlp/sempre/ParaphraseModel.java new file mode 100644 index 0000000..be7ac13 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ParaphraseModel.java @@ -0,0 +1,75 @@ +package edu.stanford.nlp.sempre; + +import java.util.HashMap; +import java.util.Map; + +import fig.basic.IOUtils; +import fig.basic.Option; +import fig.basic.MapUtils; +import fig.basic.LogInfo; + +/** + * ParaphraseModel extracts and scores paraphrasing featues from derivations. + * This model is intended to be used with FloatingParser + * + * @author Yushi Wang + */ + +public final class ParaphraseModel { + public static class Options { + @Option(gloss = "Path to file with alignment table") + public String paraphraseModelPath = "regex/regex-alignments.txt"; + } + + public static Options opts = new Options(); + + public static ParaphraseModel model; + + Map> table; + + // We should only have one paraphrase model + public static ParaphraseModel getSingleton() { + if (model == null) { + model = new ParaphraseModel(); + } + return model; + } + + private ParaphraseModel() { + table = loadParaphraseModel(opts.paraphraseModelPath); + } + + /** + * Loading paraphrase model from file + */ + private Map> loadParaphraseModel(String path) { + LogInfo.begin_track("Loading paraphrase model"); + Map> res = new HashMap>(); + for (String line: IOUtils.readLinesHard(path)) { + String[] tokens = line.split("\t"); + // double count = Double.parseDouble(tokens[2]); + MapUtils.putIfAbsent(res, tokens[0], new HashMap()); + for (String token : tokens) + LogInfo.logs("%s", token); + res.get(tokens[0]).put(tokens[1], 1.0); + /* Alignment stats don't matter right now + if(count>=threshold) { + AlignmentStats aStats = new AlignmentStats(count, Double.parseDouble(tokens[3]), Double.parseDouble(tokens[4])); + res.get(tokens[0]).put(tokens[1], aStats); + } + */ + } + LogInfo.logs("ParaphraseUtils.loadPhraseTable: number of entries=%s", res.size()); + LogInfo.end_track(); + return res; + } + + public boolean containsKey(String key) { + return table.containsKey(key); + } + + public Double get(String key, String token) { + if (!table.containsKey(key) || !table.get(key).containsKey(token)) return 0.0; + return table.get(key).get(token); + } +} diff --git a/src/edu/stanford/nlp/sempre/Parser.java b/src/edu/stanford/nlp/sempre/Parser.java index da04f15..064539d 100644 --- a/src/edu/stanford/nlp/sempre/Parser.java +++ b/src/edu/stanford/nlp/sempre/Parser.java @@ -1,16 +1,16 @@ package edu.stanford.nlp.sempre; -import edu.stanford.nlp.sempre.fbalignment.utils.DoubleContainer; import fig.basic.*; -import java.io.IOException; +import java.io.PrintWriter; import java.util.*; //////////////////////////////////////////////////////////// /** * A Parser takes an example, parses the sequence of tokens, and stores the - * derivations back in the example. + * derivations back in the example. It also computes a gradient with respect + * to some objective function. In this light, Parser it is more than a parser. * * @author Percy Liang */ @@ -18,73 +18,99 @@ public abstract class Parser { public static class Options { @Option(gloss = "For debugging, whether to print out all the predicted derivations") public boolean printAllPredictions; + @Option(gloss = "Maximal number of predictions to print") public int maxPrintedPredictions = Integer.MAX_VALUE; + @Option(gloss = "Use a coarse pass to prune the chart before full parsing") - public boolean coarsePrune = true; - @Option(gloss = "Monotonically increase the number of derivations on the beam across training iterations") - public boolean monotonicBeam = false; - @Option(gloss = "How much output to print") public int verbose = 0; - @Option(gloss = "Execute only top formula (at test time)") + public boolean coarsePrune = false; + + @Option(gloss = "How much output to print") + public int verbose = 0; + + @Option(gloss = "Execute only top formula to be cheap (hack at test time for fast demo)") public boolean executeTopFormulaOnly = false; - @Option(gloss = "Whether to evaluate with values and formulas") - public boolean evaluateValuesAndFormulas = true; - @Option(gloss = "Whether to use candidate formula parts to construct derivations") - public boolean generateFormulaCandidates = false; + + @Option(gloss = "Whether to output chart filling visualization (huge file!)") + public boolean visualizeChartFilling = false; + + @Option(gloss = "Keep this number of derivations per cell (exact use depends on the parser)") + public int beamSize = 200; + + @Option(gloss = "Whether to update based on partial reward (for learning)") + public boolean partialReward = true; + + @Option(gloss = "Whether to unroll derivation streams (applies to lazy parsers)") + public boolean unrollStream = false; } public static final Options opts = new Options(); public boolean verbose(int level) { return opts.verbose >= level; } + // Used to instantiate a parser. + public static class Spec { + public final Grammar grammar; + public final FeatureExtractor extractor; + public final Executor executor; + public final ValueEvaluator valueEvaluator; + + public Spec(Grammar grammar, FeatureExtractor extractor, Executor executor, ValueEvaluator valueEvaluator) { + this.grammar = grammar; + this.extractor = extractor; + this.executor = executor; + this.valueEvaluator = valueEvaluator; + } + } + // Inputs to the parser - public final Grammar grammar; // Specifies the set of rules + public final Grammar grammar; public final FeatureExtractor extractor; public final Executor executor; - public final FormulaRetriever formulaRetriever; + public final ValueEvaluator valueEvaluator; - //Precomputations to make looking up grammar rules faster. - ArrayList catUnaryRules; // Unary rules with category on RHS + // Precomputations to make looking up grammar rules faster. + protected List catUnaryRules; // Unary rules with category on RHS ($A => $B) + public List getCatUnaryRules() { return catUnaryRules; } - public Parser(Grammar grammar, - FeatureExtractor extractor, - Executor executor) { - this.grammar = grammar; - this.extractor = extractor; - this.executor = executor; - try { - formulaRetriever = (opts.generateFormulaCandidates) ? new FormulaRetriever(true) : null ; - } - catch(IOException e) { - throw new RuntimeException(e); - } + // TODO(joberant): move this to a separate class in charge of visualizing charts + PrintWriter chartFillOut = null; // For printing a machine-readable json file + + public Parser(Spec spec) { + this.grammar = spec.grammar; + this.extractor = spec.extractor; + this.executor = spec.executor; + this.valueEvaluator = spec.valueEvaluator; + + computeCatUnaryRules(); + LogInfo.logs("Parser: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)", catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size()); + } + + // If grammar changes, then we might need to update aspects of the parser. + public synchronized void addRule(Rule rule) { + if (rule.isCatUnary()) + catUnaryRules.add(rule); + } + + private void computeCatUnaryRules() { // Handle catUnaryRules - catUnaryRules = new ArrayList(); - Map> graph = new HashMap>(); // Node from LHS to list of rules + catUnaryRules = new ArrayList<>(); + Map> graph = new HashMap<>(); // Node from LHS to list of rules for (Rule rule : grammar.rules) if (rule.isCatUnary()) MapUtils.addToList(graph, rule.lhs, rule); // Topologically sort catUnaryRules so that B->C occurs before A->B - Map done = new HashMap(); + Map done = new HashMap<>(); for (String node : graph.keySet()) traverse(catUnaryRules, node, graph, done); - - LogInfo.logs("BeamParser: %d catUnaryRules (sorted), %d nonCatUnaryRules (in trie)", catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size()); } - public abstract int getDefaultBeamSize(); - public abstract ParserState newCoarseParserState(Params params, - Example ex); - public abstract ParserState newParserState(Params params, - Example ex, - ParserState coarseState); - - /** Helper function for transitive closure of unary rules. */ - protected void traverse(List catUnaryRules, - String node, - Map> graph, - Map done) { + // Helper function for transitive closure of unary rules. + private void traverse(List catUnaryRules, + String node, + Map> graph, + Map done) { Boolean d = done.get(node); if (Boolean.TRUE.equals(d)) return; if (Boolean.FALSE.equals(d)) @@ -97,145 +123,60 @@ public abstract class Parser { done.put(node, true); } - public void parse(Params params, Example ex) { - parse(params, ex, true); - } + // Main thing for parsers to implement. + public abstract ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts); + public Params getSearchParams(Params params) { return params; } /** * Parse the given example |ex| using the given parameters |params| - * and populate the fields of |ex| (e.g., predDerivations). NB: + * and populate the fields of |ex| (e.g., predDerivations). Note: * |ex| is modified in place. - *

- * If |execAllowed| is false, the ParserState and this function will - * both be told not to execute. A parser might rely on execution in - * order to obtain denotation features that it would use to score - * and sort predicted derivations. This means that clients passing - * |execAllowed = false| should have an internal understanding of - * the parser they are using, since they might later want to: - *

- * - Execute, - * - Possibly recompute features and scores based on the result of - * execution, - * - Possibly re-sort |ex.predDerivations|, - * - |setEvaluation()|. - *

- * As this will not be done for them by the parser. */ - public void parse(Params params, Example ex, boolean execAllowed) { + public ParserState parse(Params params, Example ex, boolean computeExpectedCounts) { + // Execute target formula (if applicable). + if (ex.targetFormula != null && ex.targetValue == null) + ex.targetValue = executor.execute(ex.targetFormula, ex.context).value; + + // Parse + StopWatch watch = new StopWatch(); + watch.start(); LogInfo.begin_track("Parser.parse: parse"); - - // Step 1: parse coarsely to see which categories should even be - // constructed. - ParserState coarseState = null; - if (opts.coarsePrune) { - LogInfo.begin_track("Parser.coarsePrune"); - coarseState = newCoarseParserState(params, ex); - coarseState.setExecAllowed(execAllowed); - coarseState.infer(); - coarseState.keepTopDownReachable(); - LogInfo.end_track(); - } - - // Step 2: parse, using the coarse information as a way to prune. - ParserState state = newParserState(params, ex, coarseState); - state.setExecAllowed(execAllowed); - if(opts.generateFormulaCandidates) - state.setFormulaSubparts(formulaRetriever.retrieveFormulas(ex)); + ParserState state = newParserState(params, ex, computeExpectedCounts); state.infer(); LogInfo.end_track(); - - ensureTargetExecuted(ex); - - if (ex.predDerivations == null) - ex.predDerivations = new ArrayList(); - - Map existingDerivs = - new HashMap(ex.predDerivations.size()); - for (Derivation deriv : ex.predDerivations) - existingDerivs.put(deriv, deriv); - - List incomingDerivs = state.getPredDerivations(); - - int reusedExecs = 0; - int notFoundExecs = 0; - int totalDerivs = 0; - for (Derivation deriv : incomingDerivs) { - // Move over useful stuff from identical existing derivations. - // Helps to avoid things like re-exec. - Derivation prev = existingDerivs.get(deriv); - if (prev == null) - notFoundExecs++; - if (prev != null && prev.isExecuted() && !deriv.isExecuted()) { - deriv.setExecResults(prev); - reusedExecs++; - } - totalDerivs++; - } - LogInfo.logs( - "Parser.parse: reusing %d/%d execs, not found %d/%d", - reusedExecs, totalDerivs, notFoundExecs, totalDerivs); - - ex.predDerivations = new ArrayList(incomingDerivs); - if (opts.monotonicBeam) { - for (Derivation deriv : incomingDerivs) - existingDerivs.remove(deriv); - ex.predDerivations.addAll(existingDerivs.keySet()); - } - - // Re-score and re-sort because we might have |setExecResults()| - // or might have a monotonic beam. - ex.rescoreAndSortPredDerivations(params); - - // Execute predicted derivations to get value. - if (execAllowed) { - LogInfo.begin_track("Parser.parse: execute"); - for (Derivation deriv : state.getPredDerivations()) { - deriv.ensureExecuted(executor); - if (opts.executeTopFormulaOnly) - break; - } - LogInfo.end_track(); - } - - // For debugging. - ex.predDerivationsAfterParse = new ArrayList(ex.predDerivations); - + watch.stop(); + state.parseTime = watch.getCurrTimeLong(); state.setEvaluation(); - if (execAllowed) - setEvaluation(ex, params); - state.clearChart(); - if(coarseState!=null) - coarseState.clearChart(); + + ex.predDerivations = state.predDerivations; + Derivation.sortByScore(ex.predDerivations); + + // Evaluate + ex.evaluation = new Evaluation(); + addToEvaluation(state, ex.evaluation); + + return state; } - // Populate the target. - protected void ensureTargetExecuted(Example ex) { - if (ex.targetFormula != null) - ex.targetValue = executor.execute(ex.targetFormula).value; - } + /** + * Compute the evaluation based on the results of parsing and add it to |evaluation| + */ + public void addToEvaluation(ParserState state, Evaluation evaluation) { + Example ex = state.ex; + List predDerivations = state.predDerivations; - public void setEvaluation(final Example ex, final Params params) { - final Evaluation eval = new Evaluation(); boolean printAllPredictions = opts.printAllPredictions; - int numCandidates = ex.predDerivations.size(); + int numCandidates = predDerivations.size(); LogInfo.begin_track_printAll("Parser.setEvaluation: %d candidates", numCandidates); - // Each derivation has a compatibility score (in [0,1]) as well as a model probability. + // Each derivation has a compatibility score (in [0, 1]) as well as a model probability. // Terminology: // True (correct): compatibility = 1 // Partial: 0 < compatibility < 1 // Wrong: compatibility = 0 - List predDerivations = ex.predDerivations; - - // Make sure at least the top derivation is executed. - for (Derivation deriv : predDerivations) { - deriv.ensureExecuted(executor); - break; - } - // Did we get the answer correct? - int correct_i = -1; // Index of first correct derivation + int correctIndex = -1; // Index of first correct derivation int correctIndexAfterParse = -1; double maxCompatibility = 0.0; double[] compatibilities = null; @@ -243,17 +184,17 @@ public abstract class Parser { compatibilities = new double[numCandidates]; for (int i = 0; i < numCandidates; i++) { Derivation deriv = predDerivations.get(i); - compatibilities[i] = deriv.compatibility = ex.targetValue.getCompatibility(deriv.value); + compatibilities[i] = deriv.compatibility; // Must be fully compatible to count as correct. - if (compatibilities[i] == 1 && correct_i == -1) - correct_i = i; - //record maximum compatibility for partial oracle + if (compatibilities[i] == 1 && correctIndex == -1) + correctIndex = i; + // record maximum compatibility for partial oracle maxCompatibility = Math.max(compatibilities[i], maxCompatibility); } // What if we only had parsed bottom up? for (int i = 0; i < numCandidates; i++) { - Derivation deriv = ex.predDerivationsAfterParse.get(i); + Derivation deriv = predDerivations.get(i); if (deriv.compatibility == 1) { correctIndexAfterParse = i; break; @@ -268,47 +209,39 @@ public abstract class Parser { deriv.prob = probs[i]; } - //evaluate values and formulas - if (opts.evaluateValuesAndFormulas) { - List> valueList = computeValueList(ex.predDerivations); - evaluateValues(eval, ex, valueList); - List, DoubleContainer>> formulaList = computeFormulaList(ex.predDerivations); - evaluateFormulas(eval, formulaList, ex); - } - // Number of derivations which have the same top score int numTop = 0; double topMass = 0; if (ex.targetValue != null) { while (numTop < numCandidates && - compatibilities[numTop] > 0.0d && - Math.abs(predDerivations.get(numTop).score - predDerivations.get(0).score) < 1e-10) { + compatibilities[numTop] > 0.0d && + Math.abs(predDerivations.get(numTop).score - predDerivations.get(0).score) < 1e-10) { topMass += probs[numTop]; numTop++; } } double correct = 0; - double partial_correct = 0; + double partCorrect = 0; if (ex.targetValue != null) { for (int i = 0; i < numTop; i++) { if (compatibilities[i] == 1) correct += probs[i] / topMass; if (compatibilities[i] > 0) - partial_correct += (compatibilities[i] * probs[i]) / topMass; + partCorrect += (compatibilities[i] * probs[i]) / topMass; } } // Print features (note this is only with respect to the first correct, is NOT the gradient). // Things are not printed if there is only partial compatability. - if (correct_i != -1 && correct != 1) { - Derivation trueDeriv = predDerivations.get(correct_i); + if (correctIndex != -1 && correct != 1) { + Derivation trueDeriv = predDerivations.get(correctIndex); Derivation predDeriv = predDerivations.get(0); - HashMap featureDiff = new HashMap(); + HashMap featureDiff = new HashMap<>(); trueDeriv.incrementAllFeatureVector(+1, featureDiff); predDeriv.incrementAllFeatureVector(-1, featureDiff); - String heading = String.format("TopTrue (%d) - Pred (%d) = Diff", correct_i, 0); - FeatureVector.logFeatureWeights(heading, featureDiff, params); + String heading = String.format("TopTrue (%d) - Pred (%d) = Diff", correctIndex, 0); + FeatureVector.logFeatureWeights(heading, featureDiff, state.params); - HashMap choiceDiff = new LinkedHashMap(); + HashMap choiceDiff = new LinkedHashMap<>(); trueDeriv.incrementAllChoices(+1, choiceDiff); predDeriv.incrementAllChoices(-1, choiceDiff); FeatureVector.logChoices(heading, choiceDiff); @@ -319,8 +252,8 @@ public abstract class Parser { Derivation deriv = predDerivations.get(i); if (compatibilities != null && compatibilities[i] == 1) { LogInfo.logs( - "True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), - Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); + "True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), + Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); } } // Partially correct @@ -328,200 +261,49 @@ public abstract class Parser { Derivation deriv = predDerivations.get(i); if (compatibilities != null && compatibilities[i] > 0 && compatibilities[i] < 1) { LogInfo.logs( - "Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), - Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); + "Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), + Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); } } // Anything that's predicted. for (int i = 0; i < predDerivations.size(); i++) { Derivation deriv = predDerivations.get(i); + // Either print all predictions or this prediction is worse by some amount. - boolean print; - if (ex.derivConstraint != null) - print = ex.derivConstraint.satisfies(ex, deriv); - else - print = printAllPredictions || ((probs[i] >= probs[0] / 2 || i < 10) && i < opts.maxPrintedPredictions); + boolean print = printAllPredictions || ((probs[i] >= probs[0] / 2 || i < 10) && i < opts.maxPrintedPredictions); if (print) { LogInfo.logs( - "Pred@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), - Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); - //LogInfo.logs("Derivation tree: %s",deriv.toRecursiveString()); + "Pred@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), + Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); + // LogInfo.logs("Derivation tree: %s", deriv.toRecursiveString()); } } - eval.add("correct", correct); - eval.add("oracle", correct_i != -1); - eval.add("partCorrect", partial_correct); - eval.add("partOracle", maxCompatibility); + evaluation.add("correct", correct); + evaluation.add("oracle", correctIndex != -1); + evaluation.add("partCorrect", partCorrect); + evaluation.add("partOracle", maxCompatibility); if (correctIndexAfterParse != -1) - eval.add("correctIndexAfterParse", correctIndexAfterParse); + evaluation.add("correctIndexAfterParse", correctIndexAfterParse); - double totBeamJump = 0.0d, totRootBeamJump = 0.0d; - double count = 1.0d; - if (!predDerivations.isEmpty()) { - for (Derivation deriv : predDerivations) { - DoubleRef beamJump = new DoubleRef(0.0d); - DoubleRef rootBeamJump = new DoubleRef(0.0d); - computeMeanBeamJump(deriv, true, beamJump, rootBeamJump); - totBeamJump += beamJump.value; - totRootBeamJump += rootBeamJump.value; - } - count = predDerivations.size(); + if (correctIndex != -1) { + evaluation.add("correctMaxBeamPosition", predDerivations.get(correctIndex).maxBeamPosition); + evaluation.add("correctMaxUnsortedBeamPosition", predDerivations.get(correctIndex).maxUnsortedBeamPosition); } - eval.add("meanBeamJump", totBeamJump / count); - eval.add("meanRootBeamJump", totRootBeamJump / count); + evaluation.add("parsed", numCandidates > 0); + evaluation.add("numCandidates", numCandidates); // From this parse + if (numCandidates > 0) + evaluation.add("parsedNumCandidates", numCandidates); - if (correct_i != -1) { - eval.add("correctMaxBeamPosition", predDerivations.get(correct_i).maxBeamPosition); - eval.add("correctMaxUnsortedBeamPosition", predDerivations.get(correct_i).maxUnsortedBeamPosition); - ex.correctMaxBeamPosition = predDerivations.get(correct_i).maxBeamPosition; - } else { - ex.correctMaxBeamPosition = -1; - } - eval.add("numCandidates", numCandidates); // From this parse - if (predDerivations.size() > 0) - eval.add("parsedNumCandidates", predDerivations.size()); + // Add parsing stats + evaluation.add(state.evaluation); - for (int i = 0; i < predDerivations.size(); i++) { - Derivation deriv = predDerivations.get(i); + // Add executor stats + for (Derivation deriv : predDerivations) { if (deriv.executorStats != null) - eval.add(deriv.executorStats); + evaluation.add(deriv.executorStats); } - // Finally, set all of these stats as the example's evaluation. - ex.setEvaluation(eval); - LogInfo.end_track(); } - - private void evaluateValues(Evaluation eval, Example ex, List> valueList) { - - double[] compatibilities = null; - if (ex.targetValue != null) { - compatibilities = new double[valueList.size()]; - for (int i = 0; i < valueList.size(); i++) { - compatibilities[i] = ex.targetValue.getCompatibility(valueList.get(i).getFirst()); - if (opts.verbose >= 3) - LogInfo.logs("evaluateValues: predValue=%s, targetValue=%s, compatibility=%s, prob=%s", valueList.get(i).getFirst(), ex.targetValue, compatibilities[i], valueList.get(i).getSecond()); - } - } - - int numTop = 0; - double topMass = 0; - while (numTop < valueList.size() && Math.abs(valueList.get(numTop).getSecond().value() - valueList.get(0).getSecond().value()) < 1e-10) { - topMass += valueList.get(numTop).getSecond().value(); - numTop++; - } - if (opts.verbose >= 3) - LogInfo.logs("evaluateValues: numTop=%s", numTop); - - - double correct = 0; - if (ex.targetValue != null) { - for (int i = 0; i < numTop; i++) { - if (compatibilities[i] == 1) - correct += valueList.get(i).getSecond().value() / topMass; - } - } - if (opts.verbose >= 3) - LogInfo.logs("evaluateValues, correct=%s", correct); - eval.add("valueCorrect", correct); - } - - private void evaluateFormulas(Evaluation eval, List, DoubleContainer>> formulaList, Example ex) { - - double[] compatibilities = null; - if (ex.targetValue != null) { - compatibilities = new double[formulaList.size()]; - for (int i = 0; i < formulaList.size(); i++) { - compatibilities[i] = ex.targetValue.getCompatibility(formulaList.get(i).getFirst().getSecond()); - if (opts.verbose >= 3) - LogInfo.logs("evaluateFormulas: predValue=%s, targetValue=%s, compatibility=%s, prob=%s", formulaList.get(i).getFirst().getSecond(), ex.targetValue, compatibilities[i], formulaList.get(i).getSecond()); - } - } - - int numTop = 0; - double topMass = 0; - while (numTop < formulaList.size() && Math.abs(formulaList.get(numTop).getSecond().value() - formulaList.get(0).getSecond().value()) < 1e-10) { - topMass += formulaList.get(numTop).getSecond().value(); - numTop++; - } - if (opts.verbose >= 3) - LogInfo.logs("evaluateFormulas: numTop=%s", numTop); - - double correct = 0; - if (ex.targetValue != null) { - for (int i = 0; i < numTop; i++) { - if (compatibilities[i] == 1) - correct += formulaList.get(i).getSecond().value() / topMass; - } - } - if (opts.verbose >= 3) - LogInfo.logs("evaluateFormulas, correct=%s", correct); - eval.add("formulaCorrect", correct); - } - - private List, DoubleContainer>> computeFormulaList(List predDerivations) { - - Map, DoubleContainer>> aggregationMap = new HashMap, DoubleContainer>>(); - //construct formula map - for (Derivation deriv : predDerivations) { - if (aggregationMap.containsKey(deriv.formula)) - aggregationMap.get(deriv.formula).getSecond().inc(deriv.prob); - else - aggregationMap.put(deriv.formula, Pair.newPair(Pair.newPair(deriv.formula, deriv.value), new DoubleContainer(deriv.prob))); - } - - List, DoubleContainer>> formulaList = new ArrayList, DoubleContainer>>(aggregationMap.values()); - double sum = 0.0; - for (Pair, DoubleContainer> pair : formulaList) - sum += pair.getSecond().value(); - if (!formulaList.isEmpty() && Math.abs(1.0 - sum) > 0.0001) - throw new RuntimeException("Sum of formulas is: " + sum); - Collections.sort(formulaList, new Pair.ReverseSecondComparator, DoubleContainer>()); - return formulaList; - } - - private List> computeValueList(List predDerivations) { - - Map> aggregationMap = new HashMap>(); - // TODO: just need a HashMap from Value to double - //construct value map - for (Derivation deriv : predDerivations) { - String strValue = deriv.value != null ? deriv.value.toString() : "-UNKNOWN-"; - if (aggregationMap.containsKey(strValue)) - aggregationMap.get(strValue).getSecond().inc(deriv.prob); - else - aggregationMap.put(strValue, new Pair(deriv.value, new DoubleContainer(deriv.prob))); - } - List> valueList = new ArrayList>(aggregationMap.values()); - double sum = 0.0; - for (Pair pair : valueList) - sum += pair.getSecond().value(); - if (!valueList.isEmpty() && Math.abs(1.0 - sum) > 0.0001) - throw new RuntimeException("Sum of values is: " + sum); - - Collections.sort(valueList, new Pair.ReverseSecondComparator()); - return valueList; - } - - private int computeMeanBeamJump(Derivation deriv, - boolean atRoot, - DoubleRef res, - DoubleRef rootRes) { - double dist = (double) (deriv.preSortBeamPosition - deriv.postSortBeamPosition); - res.value += dist; - int n = 1; - - if (deriv.children != null) - for (Derivation child : deriv.children) - n += computeMeanBeamJump(child, false, res, rootRes); - - if (atRoot) { - rootRes.value = dist; - res.value /= n; - } - - return n; - } } diff --git a/src/edu/stanford/nlp/sempre/ParserAgenda.java b/src/edu/stanford/nlp/sempre/ParserAgenda.java new file mode 100644 index 0000000..557d012 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ParserAgenda.java @@ -0,0 +1,120 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LogInfo; +import fig.basic.PriorityQueue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * Data structure for agenda in reinforcement parser + * Created by joberant on 10/20/14. + */ +public interface ParserAgenda extends Iterable { + void sort(); + boolean add(E item, double priority); + int size(); + void clear(); + E pop(); + E get(int i); + void remove(E pds, int index); +} + +class ListParserAgenda implements ParserAgenda { + + private List agenda = new ArrayList<>(); + + @Override + public void sort() { + Collections.sort(agenda); + } + + @Override + public boolean add(E item, double priority) { + return agenda.add(item); + } + + @Override + public int size() { + return agenda.size(); + } + + @Override + public void clear() { + agenda.clear(); + } + + @Override + public E pop() { + // todo - replace sort with finding max + sort(); + E pds = agenda.get(0); + remove(pds, 0); + return pds; + } + + @Override + public E get(int i) { + return agenda.get(i); + } + + @Override + public void remove(E pds, int index) { + E last = agenda.remove(agenda.size() - 1); + if (last != pds) + agenda.set(index, last); + } + + @Override + public Iterator iterator() { + return agenda.iterator(); + } +} + +class QueueParserAgenda implements ParserAgenda { + + private PriorityQueue agenda = new PriorityQueue<>(); + + @Override + public void sort() { } + + @Override + public boolean add(E item, double priority) { + return agenda.add(item, priority); + } + + @Override + public int size() { + return agenda.size(); + } + + @Override + public void clear() { + // hopefully this is never called since we sample just one + LogInfo.warning("QueueParserAgenda: clear is only called when we have more than one sample"); + while (agenda.hasNext()) + agenda.next(); + } + + @Override + public E pop() { + return agenda.next(); + } + + @Override + public E get(int i) { + throw new RuntimeException("Not supported"); + } + + @Override + public void remove(E pds, int index) { + throw new RuntimeException("Not supported"); + } + + @Override + public Iterator iterator() { + throw new RuntimeException("Not supported"); + } +} diff --git a/src/edu/stanford/nlp/sempre/ParserState.java b/src/edu/stanford/nlp/sempre/ParserState.java index c414562..75debf0 100644 --- a/src/edu/stanford/nlp/sempre/ParserState.java +++ b/src/edu/stanford/nlp/sempre/ParserState.java @@ -1,169 +1,66 @@ package edu.stanford.nlp.sempre; +import fig.basic.Fmt; import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.SetUtils; -import fig.basic.StopWatch; +import fig.basic.NumUtils; +import fig.basic.Evaluation; -import java.lang.reflect.Array; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; -/** @author Roy Frostig */ +/** + * Actually does the parsing. Main method is infer(), whose job is to fill in + * + * @author Roy Frostig + * @author Percy Liang + */ public abstract class ParserState { - // Modes: - // 1) Bool: just check if cells (cat, start, end) are reachable (to prune chart) - // 2) Full: compute everything - public enum Mode { bool, full }; + //// Input: specification of how to parse - private final Mode mode; - private final Parser parser; - private final Params params; - private final Example ex; - private final ParserState coarseState; // Used to prune (optional) - final int numTokens; + public final Parser parser; + public final Params params; + public final Example ex; + public final boolean computeExpectedCounts; // Whether we're learning - // Dynamic programming chart - // cell (start, end, category) -> list of derivations (sorted by - // decreasing score) [beam] - final Map>[][] chart; + //// Output - private boolean execAllowed; + public final List predDerivations = new ArrayList(); + public final Evaluation evaluation = new Evaluation(); - // -- Some informational stats -- - // Number of milliseconds to parse this sentence - protected long time; - // Maximum number of derivations in any chart cell prior to pruning. - protected int maxCellSize; - // Description of that cell (for debugging) - protected String maxCellDescription; - // Did any hypotheses fall off the beam? - protected boolean fallOffBeam; - protected int totalDerivsInChart=0; + // If computeExpectedCounts is true (for learning), then fill this out. + public Map expectedCounts; + public double objectiveValue; - // Parser timing - StopWatch watch; + // Statistics generated while parsing + public final int numTokens; + public long parseTime; // Number of milliseconds to parse this example + public int maxCellSize; // Maximum number of derivations in any chart cell prior to pruning. + public String maxCellDescription; // Description of that cell (for debugging) + public boolean fallOffBeam; // Did any hypotheses fall off the beam? + public int totalGeneratedDerivs; // Total number of derivations produced + public int numOfFeaturizedDerivs = 0; // Number of derivations featured - //whether to create derivations based on subparts - private boolean filterDerivationsBySubparts = false; - //subpart descriptions - private Set subParts = null; - @SuppressWarnings({"unchecked"}) - public ParserState(Mode mode, - Parser parser, - Params params, - Example ex, - ParserState coarseState) { - this.mode = mode; + @SuppressWarnings({ "unchecked" }) + public ParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) { this.parser = parser; this.params = params; this.ex = ex; - this.coarseState = coarseState; + this.computeExpectedCounts = computeExpectedCounts; this.numTokens = ex.numTokens(); - this.execAllowed = true; - - // Initialize the chart. - this.chart = (HashMap>[][]) - Array.newInstance( - HashMap.class, - numTokens, numTokens + 1); - for (int start = 0; start < numTokens; start++) { - for (int end = start + 1; end <= numTokens; end++) { - chart[start][end] = new HashMap>(); - } - } - } - - public void clearChart() { - for (int start = 0; start < numTokens; start++) { - for (int end = start + 1; end <= numTokens; end++) { - chart[start][end].clear(); - } - } } - public Mode getMode() { return mode; } - public Example getExample() { return ex; } - public ParserState getCoarseState() { return coarseState; } - public Params getModelParams() { return params; } - public long getParseTime() { return time; } - public Map>[][] getChart() { return chart; } + protected int getBeamSize() { return Parser.opts.beamSize; } - public List getPredDerivations() { - final List empty = Collections.emptyList(); - if (numTokens == 0) - return empty; - List result = chart[0][numTokens].get(Rule.rootCat); - return result == null ? empty : result; - } - - // Subclass entry point. - protected abstract void build(int start, int end); - - public void infer() { - if (numTokens == 0) - return; - - watch = new StopWatch(); - watch.start(); - if (parser.verbose(2)) - LogInfo.begin_track("ParserState.infer"); - - for (Derivation deriv : gatherTokenAndPhraseDerivations()) { - featurizeAndScoreDerivation(deriv); - addToChart(deriv); - } - - // Recurse - for (int len = 1; len <= numTokens; len++) - for (int i = 0; i + len <= numTokens; i++) - build(i, i + len); - - if (parser.verbose(2)) - LogInfo.end_track(); - watch.stop(); - time = watch.getCurrTimeLong(); - } - - int getBeamSize() { - return ex.beamSize != -1 ? ex.beamSize : parser.getDefaultBeamSize(); - } - - void addToChart(Derivation deriv) { - if(filterDerivationsBySubparts && !isValidSubpartDerivation(deriv)) - return; - MapUtils.addToList(chart[deriv.start][deriv.end], deriv.cat, deriv); - totalDerivsInChart++; - } - - @SuppressWarnings({"unchecked"}) - // TODO: hash the formulas rather than the strings. - private boolean isValidSubpartDerivation(Derivation deriv) { - if(deriv.formula==null) - return true; - if(deriv.formula instanceof PrimitiveFormula) { - ValueFormula valueFormula = (ValueFormula) deriv.formula; - if(valueFormula.value instanceof StringValue) - return true; - } - - if("(type fb:type.text)".equals(deriv.type.toString())) - return true; - boolean valid = subParts.contains(deriv.formula.toString()); - if(parser.verbose(2)) - LogInfo.logs("ParserState.isValidSubpart: formula=%s, valid=%s",deriv.formula,valid); - return valid; - } - - void setExecAllowed(boolean execAllowed) { - this.execAllowed = execAllowed; - } + // Main entry point. Should set all the output variables. + public abstract void infer(); protected void featurizeAndScoreDerivation(Derivation deriv) { - if (parser.verbose(deriv.isRoot(numTokens) ? 2 : 3)) - LogInfo.logs( - "featurizeAndScore %s %s: %s, %s", - deriv.cat, ex.spanString(deriv.start, deriv.end), deriv,deriv.rule); + if (deriv.isFeaturizedAndScored()) { + LogInfo.warnings("Derivation already featurized: %s", deriv); + return; + } // Compute features parser.extractor.extractLocal(ex, deriv); @@ -171,80 +68,76 @@ public abstract class ParserState { // Compute score deriv.computeScoreLocal(params); - if (parser.verbose(deriv.isRoot(numTokens) ? 2 : 3)) - LogInfo.logs("featurizeAndScore score=%.4f", deriv.score); + if (parser.verbose(5)) { + LogInfo.logs("featurizeAndScoreDerivation(score=%s) %s %s: %s [rule: %s]", + Fmt.D(deriv.score), deriv.cat, ex.spanString(deriv.start, deriv.end), deriv, deriv.rule); + } + numOfFeaturizedDerivs++; } - // -- Coarse state pruning -- + /** + * Prune down the number of derivations in |derivations| to the beam size. + * Sort the beam by score. + * Update beam statistics. + */ + protected void pruneCell(String cellDescription, List derivations) { + if (derivations == null) return; - // Remove any (cat, start, end) which isn't reachable from the - // (Rule.rootCat, 0, numTokens) - public void keepTopDownReachable() { - if (numTokens == 0) return; + // Update stats about cell size. + if (derivations.size() > maxCellSize) { + maxCellSize = derivations.size(); + maxCellDescription = cellDescription; + if (maxCellSize > 5000) + LogInfo.logs("ParserState.pruneCell %s: %s entries", maxCellDescription, maxCellSize); + } - Set reachable = new HashSet(); - collectReachable(reachable, Rule.rootCat, 0, numTokens); + // The extra code blocks in here that set |deriv.maxXBeamPosition| + // are there to track, over the course of parsing, the lowest + // position at which any of a derivation's constituents ever + // placed on any of the relevant beams. - // Remove all derivations associated with (cat, start, end) that aren't reachable. - for (int start = 0; start < numTokens; start++) { - for (int end = start + 1; end <= numTokens; end++) { - List toRemoveCats = new LinkedList(); - for (String cat : chart[start][end].keySet()) { - String key = catStartEndKey(cat, start, end); - if (!reachable.contains(key)) { - toRemoveCats.add(cat); - } - } - Collections.sort(toRemoveCats); - for (String cat : toRemoveCats) { - if(parser.verbose(1)) { - LogInfo.logs("Pruning chart %s(%s,%s)",cat,start,end); - } - chart[start][end].remove(cat); - } + // Max beam position (before sorting) + int i = 0; + for (Derivation deriv : derivations) { + deriv.maxUnsortedBeamPosition = i; + if (deriv.children != null) { + for (Derivation child : deriv.children) + deriv.maxUnsortedBeamPosition = Math.max(deriv.maxUnsortedBeamPosition, child.maxUnsortedBeamPosition); } - } - } - private void collectReachable(Set reachable, String cat, int start, int end) { - String key = catStartEndKey(cat, start, end); - if (reachable.contains(key)) return; - - if (!chart[start][end].containsKey(cat)) { - // This should only happen for the root when there are no parses. - return; - } - - reachable.add(key); - for (Derivation deriv : chart[start][end].get(cat)) { - for (Derivation subderiv : deriv.children) { - collectReachable(reachable, subderiv.cat, subderiv.start, subderiv.end); + if (deriv.preSortBeamPosition == -1) { + // Need to be careful to only do this once since |pruneCell()| + // might be called several times for the same beam and the + // second time around we have already sorted once. + deriv.preSortBeamPosition = i; } + i++; + } + + Derivation.sortByScore(derivations); + + // Max beam position (after sorting) + i = 0; + for (Derivation deriv : derivations) { + deriv.maxBeamPosition = i; + if (deriv.children != null) { + for (Derivation child : deriv.children) + deriv.maxBeamPosition = Math.max(deriv.maxBeamPosition, child.maxBeamPosition); + } + deriv.postSortBeamPosition = i; + i++; + } + + // Keep only the top hypotheses + int beamSize = getBeamSize(); + while (derivations.size() > beamSize) { + derivations.remove(derivations.size() - 1); + fallOffBeam = true; } } - private String catStartEndKey(String cat, int start, int end) { - return cat + ":" + start + ":" + end; - } - - // For pruning with the coarse state - protected boolean coarseAllows(Trie node, int start, int end) { - if (coarseState == null) return true; - return SetUtils.intersects( - node.cats, - coarseState.getChart()[start][end].keySet()); - } - protected boolean coarseAllows(String cat, int start, int end) { - if (coarseState == null) return true; - return coarseState.getChart()[start][end].containsKey(cat); - } - - // -- Initialization -- + // -- Base case -- public List gatherTokenAndPhraseDerivations() { - List derivs = new ArrayList(); - Example ex = getExample(); - int numTokens = ex.numTokens(); - - final List empty = Collections.emptyList(); + List derivs = new ArrayList<>(); // All tokens (length 1) for (int i = 0; i < numTokens; i++) { @@ -252,8 +145,9 @@ public abstract class ParserState { new Derivation.Builder() .cat(Rule.tokenCat).start(i).end(i + 1) .rule(Rule.nullRule) - .children(empty) + .children(Derivation.emptyList) .withStringFormulaFrom(ex.token(i)) + .canonicalUtterance(ex.token(i)) .createDerivation()); // Lemmatized version @@ -261,8 +155,9 @@ public abstract class ParserState { new Derivation.Builder() .cat(Rule.lemmaTokenCat).start(i).end(i + 1) .rule(Rule.nullRule) - .children(empty) + .children(Derivation.emptyList) .withStringFormulaFrom(ex.lemmaToken(i)) + .canonicalUtterance(ex.token(i)) .createDerivation()); } @@ -273,8 +168,9 @@ public abstract class ParserState { new Derivation.Builder() .cat(Rule.phraseCat).start(i).end(j) .rule(Rule.nullRule) - .children(empty) + .children(Derivation.emptyList) .withStringFormulaFrom(ex.phrase(i, j)) + .canonicalUtterance(ex.phrase(i, j)) .createDerivation()); // Lemmatized version @@ -282,51 +178,76 @@ public abstract class ParserState { new Derivation.Builder() .cat(Rule.lemmaPhraseCat).start(i).end(j) .rule(Rule.nullRule) - .children(empty) + .children(Derivation.emptyList) .withStringFormulaFrom(ex.lemmaPhrase(i, j)) + .canonicalUtterance(ex.phrase(i, j)) .createDerivation()); } } return derivs; } - // -- Evaluation -- - - public void setEvaluation() { - LogInfo.begin_track_printAll("ParserState.setEvaluation"); - - final Example ex = getExample(); - final Evaluation eval = new Evaluation(); - final boolean parsed = ex.predDerivations.size() > 0; - eval.add("parsed", parsed); - eval.add("numTokens", numTokens); - eval.add("maxCellSize", maxCellDescription, maxCellSize); - eval.add("fallOffBeam", fallOffBeam); - - if (getCoarseState() != null) - eval.add("coarseParseTime", getCoarseState().getParseTime()); - eval.add("parseTime", getParseTime()); - eval.add("totalDerivs", totalDerivsInChart); - - ex.setParseEvaluation(eval); + // Ensure that all the logical forms are executed and compatibilities are computed. + protected void ensureExecuted() { + LogInfo.begin_track("Parser.ensureExecuted"); + // Execute predicted derivations to get value. + for (Derivation deriv : predDerivations) { + deriv.ensureExecuted(parser.executor, ex.context); + if (ex.targetValue != null) + deriv.compatibility = parser.valueEvaluator.getCompatibility(ex.targetValue, deriv.value); + if (!computeExpectedCounts && Parser.opts.executeTopFormulaOnly) break; + } LogInfo.end_track(); } + // Add statistics to |evaluation|. + // Override if we have more statistics. + protected void setEvaluation() { + evaluation.add("numTokens", numTokens); + evaluation.add("parseTime", parseTime); + evaluation.add("maxCellSize", maxCellDescription, maxCellSize); + evaluation.add("fallOffBeam", fallOffBeam); + evaluation.add("totalDerivs", totalGeneratedDerivs); + evaluation.add("numOfFeaturizedDerivs", numOfFeaturizedDerivs); + } + + public static double compatibilityToReward(double compatibility) { + if (Parser.opts.partialReward) + return compatibility; + return compatibility == 1 ? 1 : 0; // All or nothing + } + /** - * Generate fomrula subparts and only allow derivations with these formulas - * TODO: move this logic to FormulaGenerationInfo. ParserState is generic and - * shouldn't depend on FormulaGenerationInfo. - * @param fgInfos + * Fill |counts| with the gradient with respect to the derivations + * according to a standard exponential family model over a finite set of derivations. + * Assume that everything has been executed, and compatibility has been computed. */ - public void setFormulaSubparts(List fgInfos) { - LogInfo.begin_track("Setting formulas subparts"); - filterDerivationsBySubparts=true; - subParts = new HashSet(); - for(FormulaGenerationInfo fgInfo: fgInfos) { - Formula f = fgInfo.generateFormula(); - subParts.addAll(Formulas.extractSubparts(f)); + public static void computeExpectedCounts(List derivations, Map counts) { + double[] trueScores; + double[] predScores; + + int n = derivations.size(); + if (n == 0) return; + + trueScores = new double[n]; + predScores = new double[n]; + for (int i = 0; i < n; i++) { + Derivation deriv = derivations.get(i); + double logReward = Math.log(compatibilityToReward(deriv.compatibility)); + + trueScores[i] = deriv.score + logReward; + predScores[i] = deriv.score; + } + + // Usually this happens when there are no derivations. + if (!NumUtils.expNormalize(trueScores)) return; + if (!NumUtils.expNormalize(predScores)) return; + + // Update parameters + for (int i = 0; i < n; i++) { + Derivation deriv = derivations.get(i); + double incr = trueScores[i] - predScores[i]; + deriv.incrementAllFeatureVector(incr, counts); } - LogInfo.logs("Number of subparts=%s",subParts.size()); - LogInfo.end_track(); } } diff --git a/src/edu/stanford/nlp/sempre/Parsers.java b/src/edu/stanford/nlp/sempre/Parsers.java deleted file mode 100644 index 1dd6f20..0000000 --- a/src/edu/stanford/nlp/sempre/Parsers.java +++ /dev/null @@ -1,11 +0,0 @@ -package edu.stanford.nlp.sempre; - -/** - * Util class for parsers. - * - * @author Roy Frostig - */ -public final class Parsers { - private Parsers() {} - -} diff --git a/src/edu/stanford/nlp/sempre/PrimitiveFormula.java b/src/edu/stanford/nlp/sempre/PrimitiveFormula.java index 8435697..23ba589 100644 --- a/src/edu/stanford/nlp/sempre/PrimitiveFormula.java +++ b/src/edu/stanford/nlp/sempre/PrimitiveFormula.java @@ -2,6 +2,8 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; +import java.util.List; + /** * A PrimitiveFormula represents an atomic value which is cannot be decomposed * into further symbols. Either a ValueFormula or a VariableFormula. @@ -13,4 +15,7 @@ public abstract class PrimitiveFormula extends Formula { Formula result = func.apply(this); return result == null ? this : result; } + public List mapToList(Function> func, boolean alwaysRecurse) { + return func.apply(this); + } } diff --git a/src/edu/stanford/nlp/sempre/ReinforcementParser.java b/src/edu/stanford/nlp/sempre/ReinforcementParser.java new file mode 100644 index 0000000..66d2e06 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ReinforcementParser.java @@ -0,0 +1,973 @@ +package edu.stanford.nlp.sempre; + +import com.google.common.base.Joiner; +import fig.basic.*; +import fig.exec.Execution; +import fig.prob.SampleUtils; + +import java.util.*; + +/** + * @author joberant + * Parser and learning using re-inforcement learning, that is maximizing future rewards. + * During learning we sample from the agenda that contains all possible next actions + * Then we update the gradient so that actions that were taken and led to good reward + * will get better weights than other actions + * + * At test time we simply find the highest probability parse so that is done using regular parsing + * + * We implement this using a chart and an agenda that holds the possible future actions + * + * NOTE: during search we search with the agenda and then do re-ranking + * features are separated by a "search" prefix. + * For this when we compute a derivation score on the agenda we prefix the derivation + * features with "search", and in addition, when updating the expected counts we add the + * "search" prefix. This is brittle... but at the end we score with re-ranking features + * and then the score of the derivation is \theta \times \phi as usual + */ +public class ReinforcementParser extends Parser { + public static class Options { + @Option(gloss = "Whether to do coarse pruning") + public boolean efficientCoarsePrune = true; + @Option(gloss = "Whether to do importance sampling") + public double multiplicativeBonus = 1000d; + @Option (gloss = "Number of samples") + public int numOfSamplesPerExample = 1; + @Option (gloss = "Whether to update gradient only for correct moves") + public boolean updateGradientForCorrectMovesOnly = true; + @Option (gloss = "Low probability for which we don't unroll the stream") + public double lowProb = 0.001; + @Option (gloss = "Whether to not unroll at test time") + public boolean unrollWithMaxSampler = false; + @Option (gloss = "Whether to simulate the log liklihood objective") + public boolean simulateNonRlObjective = false; + @Option (gloss = "Whether to never unroll") + public boolean noUnrolling = false; + } + public static Options opts = new Options(); + + // we assume here a binarized grammar + Map>> leftToRightSiblingMap = new HashMap<>(); + Map>> rightToLeftSiblingMap = new HashMap<>(); + Map> terminalsToRulesList = new HashMap<>(); + final CoarseParser coarseParser; + public static final String SEARCH_PREFIX = "search_"; + public final String searchPrefix; + + public ReinforcementParser(Spec spec) { + super(spec); + + coarseParser = new CoarseParser(grammar); + + // generate maps from left to right and vice versa + for (Rule rule : grammar.rules) { + if (rule.rhs.size() > 2) + throw new RuntimeException("We assume that the grammar is binarized, rule: " + rule); + if (rule.rhs.size() == 2) { + String left = rule.rhs.get(0); + String right = rule.rhs.get(1); + addToSiblingMap(left, right, rule, leftToRightSiblingMap); + addToSiblingMap(right, left, rule, rightToLeftSiblingMap); + } + if (rule.isRhsTerminals()) + MapUtils.addToList(terminalsToRulesList, Joiner.on(' ').join(rule.rhs), rule); + } + if (Parser.opts.visualizeChartFilling) + this.chartFillOut = IOUtils.openOutAppendEasy(Execution.getFile("chartfill")); + searchPrefix = opts.simulateNonRlObjective ? "" : SEARCH_PREFIX; + LogInfo.logs("ReinforcementParser(): search prefix is %s", searchPrefix); + } + + private void addToSiblingMap(String keySibling, String valueSibling, Rule rule, + Map>> siblingToSiblingMap) { + Map> valueSiblingMap = siblingToSiblingMap.get(keySibling); + if (valueSiblingMap == null) + siblingToSiblingMap.put(keySibling, valueSiblingMap = new HashMap<>()); + MapUtils.addToList(valueSiblingMap, valueSibling, rule); + } + + @Override + public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) { + if (computeExpectedCounts) { // if we learn - use sampling, otherwise, use max + // if we simulate non RL we just take the max and not do sampling + if (opts.simulateNonRlObjective) { + return new ReinforcementParserState.StateBuilder() + .parser(this) + .params(params) + .example(ex) + .samplingStrategy("max") + .computeExpectedCounts(true) + .createState(); + } else { + return (new ReinforcementParserState.StateBuilder() + .parser(this) + .params(params) + .example(ex)) + .samplingStrategy("proposal") + .computeExpectedCounts(true) + .createState(); + } + } + return (new ReinforcementParserState.StateBuilder() + .parser(this) + .params(params) + .example(ex)) + .samplingStrategy("max") + .computeExpectedCounts(false) + .createState(); + } + + @Override + public Params getSearchParams(Params params) { + return params.copyParamsByPrefix(searchPrefix); + } +} + +// Parsing by sampling items from the agenda +// Defined by: (a) sampling strategy (from agenda, from true derivation, take the max) +// (b) whether an update needs to be done +final class ReinforcementParserState extends AbstractReinforcementParserState { + + public enum NecessaryDeriv { NECESSARY_DERIV, UNNECESSARY_DERIV, UNKNOWN }; + private static final double LOG_SMALL_PROB = Math.log(ReinforcementParser.opts.lowProb); + + private final ParserAgenda agenda; + + private int completeDerivationsPushed = 0; + private int firstCorrectItem = -1; + private String samplingStrategy; + private Sampler sampler; + List correctDerivations = new ArrayList<>(); + private Map stateSequenceExpectedCounts = new HashMap<>(); + Random randGen = new Random(1); + // backpointers for remembering what derivations on the stream were popped before others + private Map, Integer>> backpointerList; + private int numItemsSampled = 0; + + public static class StateBuilder { + private ReinforcementParser parser; + private Params params; + private Example example; + private ParserState coarseState; + private String samplingStrategy = null; + private boolean computeExpectedCounts; + + public StateBuilder parser(ReinforcementParser parser) { this.parser = parser; return this; } + public StateBuilder params(Params params) { this.params = params; return this; } + public StateBuilder example(Example example) { this.example = example; return this; } + public StateBuilder samplingStrategy(String samplingStrategy) { this.samplingStrategy = samplingStrategy; return this; } + public StateBuilder computeExpectedCounts(boolean computeExpectedCounts) { this.computeExpectedCounts = computeExpectedCounts; return this; } + public ReinforcementParserState createState() { + return new ReinforcementParserState(this.parser, this.params, this.example, this.computeExpectedCounts, + this.samplingStrategy); + } + } + + // note that the sampler has a pointer to the fields of the state where they were created + private ReinforcementParserState(ReinforcementParser parser, Params params, Example ex, boolean computeExpectedCounts, + String samplingStrategy) { + super(parser, params, ex, computeExpectedCounts); + this.samplingStrategy = samplingStrategy; + backpointerList = new HashMap<>(); + agenda = samplingStrategy.equals("max") ? new QueueParserAgenda<>() : new ListParserAgenda<>(); + } + + private void clearState() { + agenda.clear(); + clearChart(); + completeDerivationsPushed = 0; + firstCorrectItem = -1; + correctDerivations.clear(); + stateSequenceExpectedCounts.clear(); + backpointerList.clear(); + numItemsSampled = 0; + } + + + protected void addToAgenda(DerivationStream derivationStream) { + addToAgenda(derivationStream, 0d); + } + + private void addToAgenda(DerivationStream derivationStream, double probSum) { + + if (!derivationStream.hasNext()) return; + + if (Parser.opts.unrollStream) { + Derivation streamHead = derivationStream.peek(); + double streamHeadScore = streamHead.score; + while (derivationStream.hasNext()) { + Derivation deriv = derivationStream.next(); + featurizeAndScoreDerivation(deriv); + if (parser.verbose(1)) { + if ((deriv.score - streamHeadScore) > 0.5) + LogInfo.logs("Unsorted stream: head=%s, head score=%s, deriv=%s, deriv score=%s", + streamHead, streamHeadScore, deriv, deriv.score); + } + addToAgendaWithScore(SingleDerivationStream.constant(deriv), deriv.score, probSum); + } + } else { + Derivation deriv = derivationStream.peek(); + featurizeAndScoreDerivation(deriv); + addToAgendaWithScore(derivationStream, deriv.score, probSum); + } + if (completeDerivationsPushed % 100 == 0) // sort the agenda from time to time + agenda.sort(); + } + + // we need to override the method because parameters are prefixed with "search_" + // this means that the score will not be the dot product and features and weights + protected void featurizeAndScoreDerivation(Derivation deriv) { + if (deriv.isFeaturizedAndScored()) { + if (parser.verbose(5)) + LogInfo.logs("Deriv already featurized=%s", deriv); + return; + } + // Compute features + parser.extractor.extractLocal(ex, deriv); + + // Compute score by adding |SEARCH_PREFIX| prefix and adding children scores + FeatureVector searchFV = deriv.addPrefixLocalFeatureVector(parser.searchPrefix); + deriv.score = searchFV.dotProduct(params); + if (deriv.children != null) + for (Derivation child : deriv.children) + deriv.score += child.score; + + if (parser.verbose(3)) + LogInfo.logs( + "featurizeAndScore(score=%s) %s %s: %s [rule: %s]", + Fmt.D(deriv.score), deriv.cat, ex.spanString(deriv.start, deriv.end), deriv, deriv.rule); + numOfFeaturizedDerivs++; + } + + private void addToAgendaWithScore(DerivationStream derivationStream, double derivScore, double probSum) { + if (derivScore == Double.NEGATIVE_INFINITY) // no need to add bad derivations + return; + Derivation deriv = derivationStream.peek(); // Score a DerivationStream based on the first item in the stream. + double priority = derivScore - (completeDerivationsPushed++ * EPSILON); + agenda.add(new PrioritizedDerivationStream(derivationStream, priority, probSum), priority); // when adding to agenda probsum is 0 + + if (parser.verbose(3)) { + LogInfo.logs("ReinforcementParser: adding to agenda: size=%s, priority=%s, deriv=%s(%s,%s), formula=%s,|pushed|=%s", + agenda.size(), priority, deriv.cat, deriv.start, deriv.end, deriv.formula, completeDerivationsPushed); + } + } + + public boolean continueParsing() { + if (agenda.size() == 0) { + LogInfo.log("Agenda is empty"); + return false; + } + + return chart[0][numTokens].get(Rule.rootCat) == null || + chart[0][numTokens].get(Rule.rootCat).size() < getBeamSize(); + } + + public void infer() { + if (numTokens == 0) + return; + + ReinforcementParserState oracleState = null; + expectedCounts = new HashMap<>(); + if (computeExpectedCounts && !ReinforcementParser.opts.simulateNonRlObjective) { // when updating params we first find a correct derivation to set the oracle sampler + // TODO(jonathan): move to ReinforcementParser, not ParserState + LogInfo.begin_track("Finding oracle derivation"); + oracleState = new StateBuilder() + .parser(this.parser) + .params(this.params) + .example(this.ex) + .samplingStrategy("agenda") + .computeExpectedCounts(false).createState(); // update params is false preventing an infinite loop + oracleState.infer(); + LogInfo.end_track(); + if (oracleState.correctDerivations.isEmpty()) { + LogInfo.logs("No oracle derivation found"); + return; + } + } + createSampler(oracleState); // we can only create the sampler after we have the oracle derivation + + LogInfo.begin_track("Coarse parsing"); + coarseParserState = null; + if (ReinforcementParser.opts.efficientCoarsePrune) + coarseParserState = coarseParser.getCoarsePrunedChart(ex); + LogInfo.end_track(); + + // draw samples to compute gradient and expected reward + int i = 0; + for ( ; i < (computeExpectedCounts ? ReinforcementParser.opts.numOfSamplesPerExample : 1); ++i) { + LogInfo.begin_track("ReinforcementParserState.infer, sample=%s", i); + if (i > 0) + clearState(); + inferBySampling(); + LogInfo.end_track(); + } + + // Compute gradient + setPredDerivations(); + if (computeExpectedCounts) { + expectedCounts = ReinforcementUtils.multiplyDoubleMap(expectedCounts, 1d / i); + if (parser.verbose(3)) + SempreUtils.logMap(expectedCounts, "averaged gradient"); + } + objectiveValue /= i; + if (parser.verbose(3)) + LogInfo.logs("Expected reward (across samples) = %s", objectiveValue); + + visualizeChart(); + } + + private void inferBySampling() { + + // add to chart the token and phrase parts + for (Derivation deriv : gatherTokenAndPhraseDerivations()) + addToAgenda(SingleDerivationStream.constant(deriv)); + // add to agenda unaries where RHS is just terminals + for (DerivationStream derivStream : gatherRhsTerminalsDerivations()) + addToAgenda(derivStream); + + ensureExecuted(); + + while (continueParsing()) { + + unrollHighProbStreams(); + Pair pdsAndProbability = sampler.sample(); + + DerivationStream sampledDerivations = pdsAndProbability.getFirst().derivStream; + Derivation sampledDerivation = sampledDerivations.next(); + updateBackpointers(sampledDerivations, sampledDerivation); // to be able to get all correct actions + numItemsSampled++; + + assert sampledDerivation.isFeaturizedAndScored() : "top derivation is not featurized and scored: " + sampledDerivation; + assert Math.abs(sampledDerivation.score - pdsAndProbability.getFirst().priority) < 1e-4 : + sampledDerivation.score + " != " + pdsAndProbability.getFirst().priority; + + if (parser.verbose(2)) { + LogInfo.begin_track("Item %d (|agenda|=%d), priority %s: |item|=%s -> %s %s %s [%s], prob=%s", + numItemsSampled, agenda.size() + 1, Fmt.D(pdsAndProbability.getFirst().priority), sampledDerivations.estimatedSize(), + sampledDerivation.cat, ex.spanString(sampledDerivation.start, sampledDerivation.end), sampledDerivation, + sampledDerivation.rule, pdsAndProbability.getSecond()); + } + + // handle root derivations - get compatibility and record number of compatible derivations + recordRootDerivationStats(ex, numItemsSampled, sampledDerivation); + if (computeExpectedCounts) { + Map counts = new HashMap<>(); + // add the feature vector and subtract for the time it was in the agenda unless has negative probability + if (pdsAndProbability.getSecond() > -0.0001) + sampledDerivation.incrementLocalFeatureVector(1 - pdsAndProbability.getFirst().probSum, counts); + else + sampledDerivation.incrementLocalFeatureVector(-pdsAndProbability.getFirst().probSum, counts); + if (parser.verbose(3)) + SempreUtils.logMap(counts, "agenda item gradient"); + ReinforcementUtils.addToDoubleMap(stateSequenceExpectedCounts, counts, parser.searchPrefix); // upate the gradient incrementally + } + // only after update of params we can change the chart and the agenda + if (addToBoundedChart(sampledDerivation)) { + if (parser.verbose(5)) + LogInfo.logs("ReinforcementParserState.infer: adding to chart %s(%s,%s) formula=%s", + sampledDerivation.cat, sampledDerivation.start, sampledDerivation.end, sampledDerivation.formula); + combineWithChartDerivations(sampledDerivation); + } + addToAgenda(sampledDerivations); + if (parser.verbose(2)) + LogInfo.end_track(); + } + + finalizeSearchExpectedCounts(); // gradient for remaining agenda items + rerankRootDerivations(); // last action + if (computeExpectedCounts) { + computeGradient(); + } + } + + private void unrollHighProbStreams() { + + if (ReinforcementParser.opts.noUnrolling) return; + if (samplingStrategy.equals("max") && !ReinforcementParser.opts.unrollWithMaxSampler) return; + + sampler.unroll(); // if multiplicative, then unroll oracle stuff + + if (parser.verbose(3)) + LogInfo.begin_track("Unrolling high probability streams"); + // get max agenda score + double maxScore = Double.NEGATIVE_INFINITY; + for (PrioritizedDerivationStream pds : agenda) { + double agendaScore = pds.getScore(); + if (maxScore < agendaScore) + maxScore = agendaScore; + } + + if (parser.verbose(3)) + LogInfo.logs("unrollHighProbStreams(): |agenda|=%s, max=%s", agenda.size(), maxScore); + + List> derivsToAdd = new ArrayList<>(); + List indicesToRemove = new ArrayList<>(); + for (int i = 0; i < agenda.size(); ++i) { + PrioritizedDerivationStream pds = agenda.get(i); + boolean modified = false; + while (pds.derivStream.hasNext() && pds.derivStream.estimatedSize() > 1 && + isHighProbStream(pds.derivStream, maxScore, pds.derivStream.estimatedSize())) { + modified = true; + Derivation nextDeriv = pds.derivStream.next(); + updateBackpointers(pds.derivStream, nextDeriv); + + DerivationStream derivStream = SingleDerivationStream.constant(nextDeriv); + if (parser.verbose(3)) { + Derivation deriv = derivStream.peek(); + LogInfo.logs("unrollHighProbStreams(): add deriv=%s(%s,%s) [%s] score=%s, |stream|=%s", + deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.score, pds.derivStream.estimatedSize()); + } + derivsToAdd.add(Pair.newPair(derivStream, pds.probSum)); + if (pds.derivStream.hasNext()) + featurizeAndScoreDerivation(pds.derivStream.peek()); + } + if (modified) { + indicesToRemove.add(i); + derivsToAdd.add(Pair.newPair(pds.derivStream, pds.probSum)); + } + } + + // remove - need to make sure indices don't change due to removal so go from end to start + for (int i = indicesToRemove.size() - 1; i >= 0; --i) + agenda.remove(agenda.get(indicesToRemove.get(i)), indicesToRemove.get(i)); + + // add + for (Pair pair : derivsToAdd) + addToAgenda(pair.getFirst(), pair.getSecond()); + + if (parser.verbose(3)) + LogInfo.logs("unrollHighProbStreams(): |agenda|=%s", agenda.size()); + if (parser.verbose(3)) + LogInfo.end_track(); + } + + private boolean isHighProbStream(DerivationStream derivStream, double maxScore, int estimatedSize) { + Derivation deriv = derivStream.peek(); + double gapFromMax = deriv.score - maxScore; + double threshold = LOG_SMALL_PROB - Math.log(estimatedSize); + if (parser.verbose(3)) + LogInfo.logs("isHighProbStream(): gapFromMax=%s, threshold=%s, deriv=%s(%s,%s) %s |stream|=%s", gapFromMax, threshold, + deriv.cat, deriv.start, deriv.end, deriv.formula, derivStream.estimatedSize()); + + return gapFromMax > threshold; + } + + // recompute score using dot product of features and re-ranking feature weights + private void rerankRootDerivations() { + setPredDerivations(); + for (Derivation rootDeriv : predDerivations) { + double oldScore = rootDeriv.score; + rootDeriv.computeScore(params); + if (parser.verbose(3)) + LogInfo.logs("ReinforcementParser.rerankRootDerivations: deriv=%s, old=%s, new=%s", rootDeriv, oldScore, rootDeriv.score); + } + Derivation.sortByScore(predDerivations); + } + + private void updateBackpointers(DerivationStream stream, Derivation sampledDeriv) { + Pair, Integer> pair = backpointerList.get(sampledDeriv.creationIndex); + if (!stream.hasNext()) return; + + if (pair == null) { + ArrayList list = new ArrayList<>(); + list.add(sampledDeriv); + backpointerList.put(sampledDeriv.creationIndex, pair = Pair.newPair(list, 0)); + } + List list = pair.getFirst(); + Derivation nextDeriv = stream.peek(); + list.add(nextDeriv); + backpointerList.put(nextDeriv.creationIndex, Pair.newPair(pair.getFirst(), list.size() - 1)); + } + + private double computeExpectedReward(List predDerivations, double[] probs) { + double rewardExpectation = 0d; + for (int i = 0; i < predDerivations.size(); ++i) { + rewardExpectation += probs[i] * compatibilityToReward(predDerivations.get(i).compatibility); + } + return rewardExpectation; + } + + // q - proposal distribution + // pi = model distribution + private void computeGradient() { + if (predDerivations.isEmpty()) return; + + double[] qDist = sampler.getDerivDistribution(predDerivations); + double[] piDist = ReinforcementUtils.expNormalize(predDerivations); + // compute E_q(R(d)) + LogInfo.begin_track("Computing gradient"); + double rewardExpectation = computeExpectedReward(predDerivations, qDist); + + // compute E_q(\phi(d) R(d)) and E_pi(\phi(d)) + Map featureExpectation = new HashMap<>(), rewardInfusedFeatureExpectation = new HashMap<>(); + for (int i = 0; i < predDerivations.size(); ++i) { + Derivation deriv = predDerivations.get(i); + deriv.incrementAllFeatureVector(piDist[i], featureExpectation); + deriv.incrementAllFeatureVector(qDist[i] * compatibilityToReward(deriv.compatibility), rewardInfusedFeatureExpectation); + } + // final gradient computation + Map sampleCounts = new HashMap<>(); + if (ReinforcementParser.opts.simulateNonRlObjective) { + ParserState.computeExpectedCounts(predDerivations, sampleCounts); + } else { + sampleCounts = ReinforcementUtils.multiplyDoubleMap(stateSequenceExpectedCounts, rewardExpectation); + SempreUtils.addToDoubleMap(sampleCounts, rewardInfusedFeatureExpectation); + ReinforcementUtils.subtractFromDoubleMap(sampleCounts, ReinforcementUtils.multiplyDoubleMap(featureExpectation, rewardExpectation)); + } + SempreUtils.addToDoubleMap(expectedCounts, sampleCounts); + + double sum = 0d; + for (String key : sampleCounts.keySet()) { + double value = sampleCounts.get(key); + if (parser.verbose(3)) + LogInfo.logs("feature=%s, value=%s", key, value); + sum += value * value; + } + LogInfo.logs("L2 norm: %s", Math.sqrt(sum)); + LogInfo.end_track(); + } + + private void createSampler(ReinforcementParserState oracleState) { + if ("proposal".equals(samplingStrategy)) { + if (oracleState == null) + throw new RuntimeException("missing oracle state"); + this.sampler = new MultiplicativeProposalSampler(oracleState); + } else if ("max".equals(samplingStrategy)) { + this.sampler = new MaxSampler(); + } else if ("agenda".equals(samplingStrategy) || samplingStrategy == null) // default + this.sampler = new AgendaSampler(); + } + + // info for visualizing chart + void visualizeChart() { + if (parser.chartFillOut != null && Parser.opts.visualizeChartFilling) { + parser.chartFillOut.println(Json.writeValueAsStringHard(new ChartFillingData(ex.id, chartFillingList, + ex.utterance, ex.numTokens()))); + parser.chartFillOut.flush(); + } + } + + // go over all agenda items that we did not subtract counts for and finalize + private void finalizeSearchExpectedCounts() { + if (ReinforcementParser.opts.simulateNonRlObjective) return; + if (!computeExpectedCounts) return; + Map counts = new HashMap<>(); + for (PrioritizedDerivationStream pds : agenda) { + pds.derivStream.peek().incrementLocalFeatureVector(pds.probSum, counts); + } + if (parser.verbose(3)) { + SempreUtils.logMap(counts, "subtracted"); + } + ReinforcementUtils.subtractFromDoubleMap(stateSequenceExpectedCounts, counts, parser.searchPrefix); + if (parser.verbose(3)) { + SempreUtils.logMap(stateSequenceExpectedCounts, "final search gradient"); + } + } + + private void recordRootDerivationStats(Example ex, int numItemsSampled, Derivation sampledDerivation) { + if (!sampledDerivation.isRoot(ex.numTokens())) return; + + sampledDerivation.ensureExecuted(parser.executor, ex.context); + if (ex.targetValue != null) + sampledDerivation.compatibility = parser.valueEvaluator.getCompatibility(ex.targetValue, sampledDerivation.value); + if (Parser.opts.partialReward ? (sampledDerivation.compatibility > 0) : (sampledDerivation.compatibility == 1)) { + if (parser.verbose(2)) + LogInfo.logs("Top-level %s: reward = %s", numItemsSampled, sampledDerivation.compatibility); + // put in position 0 the derivation with best compatibility + correctDerivations.add(sampledDerivation); + if (correctDerivations.get(0).compatibility < sampledDerivation.compatibility) { + Collections.swap(correctDerivations, 0, correctDerivations.size() - 1); + } + + if (firstCorrectItem == -1) + firstCorrectItem = numItemsSampled; + } + } + + public void setEvaluation() { + LogInfo.begin_track_printAll("ReinforcementParserParserState.setEvaluation"); + super.setEvaluation(); + + if (coarseParserState != null) + evaluation.add("coarseParseTime", coarseParserState.getCoarseParseTime()); + if (firstCorrectItem != -1) + evaluation.add("firstCorrectItem", firstCorrectItem); + LogInfo.end_track(); + } + + // Defines how to sample the next state + abstract class Sampler { + // returns a derivation stream with the sample probability + public abstract Pair sample(); + public abstract double[] getDerivDistribution(List rootDerivs); + public abstract void unroll(); + + // go over agenda and update the probability sum for gradient computation before sampling + public void updateProbSum(double[] modelProbs) { + for (int i = 0; i < agenda.size(); ++i) { + PrioritizedDerivationStream pds = agenda.get(i); + pds.addProb(modelProbs[i]); + if (parser.verbose(3)) + LogInfo.logs("updateProbSum(): deriv=%s, probSum=%s", pds.derivStream.peek(), pds.probSum); + } + } + } + + // sample from agenda based on agenda scores + class AgendaSampler extends Sampler { + + @Override + + public Pair sample() { + + double[] modelProbs = ReinforcementUtils.expNormalize(agenda); + if (computeExpectedCounts)// compute probability sum before sampling for gradient computation (easier before sampling) + updateProbSum(modelProbs); + + int sampledIndex = SampleUtils.sampleMultinomial(randGen, modelProbs); + PrioritizedDerivationStream pds = agenda.get(sampledIndex); + double prob = modelProbs[sampledIndex]; + agenda.remove(pds, sampledIndex); + return Pair.newPair(pds, prob); + } + + @Override + public double[] getDerivDistribution(List rootDerivs) { + return ReinforcementUtils.expNormalize(rootDerivs); + } + + @Override + public void unroll() { } + } + + class MaxSampler extends Sampler { + + @Override + public Pair sample() { + PrioritizedDerivationStream pds = agenda.pop(); + return Pair.newPair(pds, 1d); + } + + @Override + public double[] getDerivDistribution(List rootDerivs) { + double[] res = new double[rootDerivs.size()]; + Arrays.fill(res, 0d); + res[0] = 1d; + return res; + } + + @Override + public void unroll() { } + } + + // sample categories and spans that are in a correct derivation + class OracleInfo { + + private List necessaryDerivInfos; // info for derivations necessary to generate oracle derivations + private List oracleDerivInfos; // info for oracle derivations + private Map, Integer>> backPointers; + private NecessaryDeriv[] necessaryDerivsCache; // memorizing whether a deriv is necessary or not + long firstCorrectDerivNumber = -1; // offset for necessaryDerivs + + // we sample from the end to the start + public OracleInfo(ReinforcementParserState oracleState) { + if (oracleState == null) throw new RuntimeException("oracle state is null"); + this.necessaryDerivInfos = new ArrayList<>(); + this.oracleDerivInfos = new ArrayList<>(); + if (!oracleState.correctDerivations.isEmpty()) { + Collections.sort(oracleState.correctDerivations, new CorrectDerivationComparator()); + this.backPointers = oracleState.backpointerList; + Derivation oracleDeriv = oracleState.correctDerivations.get(0); + this.firstCorrectDerivNumber = oracleDeriv.creationIndex; + + LogInfo.logs("OracleSampler: deriv=%s, comp=%s", oracleDeriv, oracleDeriv.compatibility); + if (SemanticFn.opts.trackLocalChoices) { + HashMap choiceDiff = new LinkedHashMap<>(); + oracleDeriv.incrementAllChoices(+1, choiceDiff); + FeatureVector.logChoices("Oracle deriv", choiceDiff); + } + populateCorrectDerivations(oracleDeriv); + + if (parser.verbose(2)) { + LogInfo.begin_track("OracleSampler: necessary infos:"); + for (DerivInfo necessaryInfo : necessaryDerivInfos) LogInfo.log(necessaryInfo); + LogInfo.end_track(); + LogInfo.begin_track("OracleSampler: oracle infos:"); + for (DerivInfo oracleInfo : oracleDerivInfos) LogInfo.log(oracleInfo); + LogInfo.end_track(); + } + } + } + + private void populateCorrectDerivations(Derivation oracleDeriv) { + // add derivation info and also all upstream derivations + if (parser.verbose(4)) + LogInfo.logs("populateCorrectDerivations(): oracle deriv: %s", oracleDeriv); + + Pair, Integer> listAndIndex = this.backPointers.get(oracleDeriv.creationIndex); + if (listAndIndex != null) { + for (int i = listAndIndex.getSecond() - 1; i >= 0; i--) { + Derivation deriv = listAndIndex.getFirst().get(i); + if (parser.verbose(4)) + LogInfo.logs("populateCorrectDerivations(): necessary deriv: %s", deriv); + DerivInfo derivInfo = new DerivInfo(deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.rule); + if (!necessaryDerivInfos.contains(derivInfo)) + necessaryDerivInfos.add(derivInfo); + } + } + DerivInfo derivInfo = new DerivInfo(oracleDeriv.cat, oracleDeriv.start, oracleDeriv.end, oracleDeriv.formula, oracleDeriv.rule); + if (!oracleDerivInfos.contains(derivInfo)) { + necessaryDerivInfos.add(derivInfo); + oracleDerivInfos.add(derivInfo); + } + + // recurse + for (Derivation child : oracleDeriv.children) { + populateCorrectDerivations(child); + } + } + + // checking if derivation is necessary using the cache + protected boolean isNecessaryDeriv(Derivation deriv) { + + if (necessaryDerivInfos.isEmpty()) return false; + if (necessaryDerivsCache == null) { + necessaryDerivsCache = new NecessaryDeriv[200000]; + Arrays.fill(necessaryDerivsCache, NecessaryDeriv.UNKNOWN); + } + int index = (int) (deriv.creationIndex - firstCorrectDerivNumber); + if (index < 0) throw new RuntimeException("Negative index - correct index larger than deriv number"); + if (index >= 200000) { + LogInfo.warnings("isNecessaryDeriv(): index larger than 200000: %s", index); + return necessaryDerivInfos.contains(new DerivInfo(deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.rule)); + } + if (necessaryDerivsCache[index] == NecessaryDeriv.UNNECESSARY_DERIV) + return false; + if (necessaryDerivsCache[index] == NecessaryDeriv.NECESSARY_DERIV) + return true; + // unknown + boolean res = necessaryDerivInfos.contains(new DerivInfo(deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.rule)); + necessaryDerivsCache[index] = res ? NecessaryDeriv.NECESSARY_DERIV : NecessaryDeriv.UNNECESSARY_DERIV; + return res; + } + } + + class MultiplicativeProposalSampler extends Sampler { + + private double bonus; + private OracleInfo oracleInfo; + + public MultiplicativeProposalSampler(ReinforcementParserState oracleState) { + oracleInfo = new OracleInfo(oracleState); + bonus = ReinforcementParser.opts.multiplicativeBonus; + LogInfo.logs("Bonus=%s", bonus); + } + + // We assume that oracle stuff has been unrolled + @Override + public Pair sample() { + + double[] modelProbs = ReinforcementUtils.expNormalize(agenda); + double[] samplerProbs = getUnnormalizedAgendaDistribution(); + if (!NumUtils.expNormalize(samplerProbs)) throw new RuntimeException("Normalization failed" + Arrays.toString(samplerProbs)); + + int sampledIndex = ReinforcementUtils.sampleIndex(randGen, samplerProbs); + PrioritizedDerivationStream pds = agenda.get(sampledIndex); + double prob = samplerProbs[sampledIndex]; + + if (parser.verbose(3)) { + Derivation deriv = pds.derivStream.peek(); + if (oracleInfo.oracleDerivInfos.contains(new DerivInfo(deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.rule))) + LogInfo.logs("MultiplicativeProposalSampler.sample(): Sampled from correct!, prob=%s", prob); + else + LogInfo.logs("MultiplicativeProposalSampler.sample(): Sampled from incorrect!, prob=%s", prob); + } + boolean returnProb = true; + + // whether to update only for correct moves or not hack + if (computeExpectedCounts && ReinforcementParser.opts.updateGradientForCorrectMovesOnly) { + Derivation deriv = pds.derivStream.peek(); + DerivInfo derivInfo = new DerivInfo(deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.rule); + if (oracleInfo.oracleDerivInfos.contains(derivInfo)) + updateProbSum(modelProbs); + else returnProb = false; + + if (parser.verbose(3)) { + LogInfo.logs("Updating gradient=%s", returnProb); + } + } else { + if (computeExpectedCounts) // compute probability sum before sampling for gradient computation (easier before sampling) + updateProbSum(modelProbs); + } + + agenda.remove(pds, sampledIndex); + return Pair.newPair(pds, returnProb ? prob : -1d); + } + + @Override + public void unroll() { + + if (parser.verbose(3)) + LogInfo.begin_track("MultiplicativeBonusSampler.unroll()"); + + List> derivsToAdd = new ArrayList<>(); + List indicesToRemove = new ArrayList<>(); + + for (int i = 0; i < agenda.size(); ++i) { + PrioritizedDerivationStream pds = agenda.get(i); + boolean modified = false; + while (pds.derivStream.hasNext() && pds.derivStream.estimatedSize() > 1 && oracleInfo.isNecessaryDeriv(pds.derivStream.peek())) { + modified = true; + Derivation nextDeriv = pds.derivStream.next(); + DerivationStream newDerivStream = SingleDerivationStream.constant(nextDeriv); + if (parser.verbose(3)) { + Derivation deriv = newDerivStream.peek(); + LogInfo.logs("MultiplicativeSampler.unroll(): add necessary deriv=%s(%s,%s) [%s] score=%s, |stream|=%s, creationIndex=%s", + deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.score, pds.derivStream.estimatedSize(), deriv.creationIndex); + } + derivsToAdd.add(Pair.newPair(newDerivStream, pds.probSum)); + if (pds.derivStream.hasNext()) + featurizeAndScoreDerivation(pds.derivStream.peek()); + } + if (modified) { + indicesToRemove.add(i); + derivsToAdd.add(Pair.newPair(pds.derivStream, pds.probSum)); + } + } + // remove - need to make sure indices don't change due to removal so go from end to start + for (int i = indicesToRemove.size() - 1; i >= 0; --i) + agenda.remove(agenda.get(indicesToRemove.get(i)), indicesToRemove.get(i)); + // add + for (Pair pair: derivsToAdd) + addToAgenda(pair.getFirst(), pair.getSecond()); + if (parser.verbose(3)) + LogInfo.end_track(); + } + + private double[] getUnnormalizedAgendaDistribution() { + double[] probs = new double[agenda.size()]; + for (int i = 0; i < agenda.size(); ++i) { + Derivation d = agenda.get(i).derivStream.peek(); + probs[i] = d.score; + // we assume all necessary things have been unrolled already so no need to handle that + if (oracleInfo.oracleDerivInfos.contains(new DerivInfo(d.cat, d.start, d.end, d.formula, d.rule))) { + probs[i] += bonus; + } + } + return probs; + } + + @Override + public double[] getDerivDistribution(List rootDerivs) { + double[] res = new double[rootDerivs.size()]; + for (int i = 0; i < rootDerivs.size(); ++i) { + Derivation rootDeriv = rootDerivs.get(i); + res[i] = rootDeriv.score + bonus * rootDeriv.compatibility; + } + NumUtils.expNormalize(res); + return res; + } + } + + public static class CorrectDerivationComparator implements Comparator { + @Override + public int compare(Derivation deriv1, Derivation deriv2) { + if (deriv1.compatibility > deriv2.compatibility) return -1; + if (deriv1.compatibility < deriv2.compatibility) return +1; + + boolean deriv1Join = containsJoin(deriv1); + boolean deriv2Join = containsJoin(deriv2); + if (deriv1Join && !deriv2Join) return -1; + if (!deriv1Join && deriv2Join) return +1; + // by score + if (deriv1.score > deriv2.score) return -1; + if (deriv1.score < deriv2.score) return +1; + // Ensure reproducible randomness + if (deriv1.creationIndex < deriv2.creationIndex) return -1; + if (deriv1.creationIndex > deriv2.creationIndex) return +1; + return 0; + } + + private boolean containsJoin(Derivation d) { + SemanticFn semanticFn = d.rule.getSem(); + if (semanticFn != null) { + if (semanticFn instanceof JoinFn) + return true; + } + for (Derivation child : d.children) { + if (containsJoin(child)) + return true; + } + return false; + } + } +} + +//holds the stream, the priority, and a probability sum to make grdient computation efficient +class PrioritizedDerivationStream implements Comparable, HasScore { + public final DerivationStream derivStream; + public final double priority; + public double probSum; + + PrioritizedDerivationStream(DerivationStream derivStream, double priority, double probSum) { + this.derivStream = derivStream; + this.priority = priority; + this.probSum = probSum; + } + + @Override + public int compareTo(PrioritizedDerivationStream o) { + if (this.priority > o.priority) return -1; + if (this.priority < o.priority) return +1; + return 0; + } + + public double getScore() { return derivStream.peek().score; } + public void addProb(double prob) { probSum += prob; } +} + +class DerivInfo { + public final String cat; + public final int start; + public final int end; + public final Formula formula; + public final Rule rule; + + DerivInfo(String cat, int start, int end, Formula formula, Rule rule) { + this.cat = cat; + this.start = start; + this.end = end; + this.formula = formula; + this.rule = rule; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DerivInfo derivInfo = (DerivInfo) o; + + if (end != derivInfo.end) return false; + if (start != derivInfo.start) return false; + if (cat != null ? !cat.equals(derivInfo.cat) : derivInfo.cat != null) return false; + if (formula != null ? !formula.equals(derivInfo.formula) : derivInfo.formula != null) return false; + return !(rule != null ? !rule.equals(derivInfo.rule) : derivInfo.rule != null); + + } + + @Override + public int hashCode() { + int result = cat != null ? cat.hashCode() : 0; + result = 31 * result + start; + result = 31 * result + end; + result = 31 * result + (formula != null ? formula.hashCode() : 0); + result = 31 * result + (rule != null ? rule.hashCode() : 0); + return result; + } + + public String toString() { + return cat + "(" + start + "," + end + ") " + formula.toString(); + } +} + diff --git a/src/edu/stanford/nlp/sempre/ReinforcementUtils.java b/src/edu/stanford/nlp/sempre/ReinforcementUtils.java new file mode 100644 index 0000000..175d4e9 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ReinforcementUtils.java @@ -0,0 +1,148 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.MapUtils; +import fig.basic.NumUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +/** + * Utils for ReinforcementParser + */ +public final class ReinforcementUtils { + private ReinforcementUtils() { } + + // add to double map after adding prefix to all keys + public static void addToDoubleMap(Map mutatedMap, Map addedMap, String prefix) { + for (String key : addedMap.keySet()) + MapUtils.incr(mutatedMap, prefix + key, addedMap.get(key)); + } + + public static void subtractFromDoubleMap(Map mutatedMap, Map subtractedMap) { + for (String key : subtractedMap.keySet()) + MapUtils.incr(mutatedMap, key, -1 * subtractedMap.get(key)); + } + // subtract from double map after adding prefix to all keys + public static void subtractFromDoubleMap(Map mutatedMap, Map subtractedMap, String prefix) { + for (String key : subtractedMap.keySet()) + MapUtils.incr(mutatedMap, prefix + key, -1 * subtractedMap.get(key)); + } + + public static Map multiplyDoubleMap(Map map, double factor) { + Map res = new HashMap<>(); + for (Map.Entry entry: map.entrySet()) + res.put(entry.getKey(), entry.getValue() * factor); + return res; + } + + public static int sampleIndex(Random rand, List scorables, double denominator) { + double randD = rand.nextDouble(); + double sum = 0; + + for (int i = 0; i < scorables.size(); ++i) { + HasScore pds = scorables.get(i); + double prob = computeProb(pds, denominator); + sum += prob; + if (randD < sum) { + return i; + } + } + throw new RuntimeException(sum + " < " + randD); + } + + public static int sampleIndex(Random rand, double[] scores, double denominator) { + double randD = rand.nextDouble(); + double sum = 0; + + for (int i = 0; i < scores.length; ++i) { + double pds = scores[i]; + double prob = computeProb(pds, denominator); + sum += prob; + if (randD < sum) { + return i; + } + } + throw new RuntimeException(sum + " < " + randD); + } + + public static int sampleIndex(Random rand, double[] probs) { + double randD = rand.nextDouble(); + double sum = 0; + + for (int i = 0; i < probs.length; ++i) { + sum += probs[i]; + if (randD < sum) return i; + } + throw new RuntimeException(sum + " < " + randD); + } + + public static double computeProb(HasScore deriv, double denominator) { + double prob = Math.exp(deriv.getScore() - denominator); + if (prob < -0.0001 || prob > 1.0001) + throw new RuntimeException("Probability is out of range, prob=" + prob + + ",score=" + deriv.getScore() + ", denom=" + denominator); + return prob; + } + + public static double computeProb(double score, double denominator) { + double prob = Math.exp(score - denominator); + if (prob < -0.0001 || prob > 1.0001) + throw new RuntimeException("Probability is out of range, prob=" + prob + + ",score=" + score + ", denom=" + denominator); + return prob; + } + + public static double computeLogExpSum(List scorables) { + double sum = Double.NEGATIVE_INFINITY; + for (HasScore scorable : scorables) { + sum = NumUtils.logAdd(sum, scorable.getScore()); + } + return sum; + } + + public static double[] expNormalize(List scorables) { + // Input: log probabilities (unnormalized too) + // Output: normalized probabilities + // probs actually contains log probabilities; so we can add an arbitrary constant to make + // the largest log prob 0 to prevent overflow problems + double[] res = new double[scorables.size()]; + double max = Double.NEGATIVE_INFINITY; + for (int i = 0; i < scorables.size(); i++) + max = Math.max(max, scorables.get(i).getScore()); + if (Double.isInfinite(max)) + throw new RuntimeException("Scoreables is probably empty"); + for (int i = 0; i < scorables.size(); i++) + res[i] = Math.exp(scorables.get(i).getScore() - max); + NumUtils.normalize(res); + return res; + } + + public static double[] expNormalize(ParserAgenda scorables) { + // Input: log probabilities (unnormalized too) + // Output: normalized probabilities + // probs actually contains log probabilities; so we can add an arbitrary constant to make + // the largest log prob 0 to prevent overflow problems + double[] res = new double[scorables.size()]; + double max = Double.NEGATIVE_INFINITY; + + + for (HasScore scorable : scorables) + max = Math.max(max, scorable.getScore()); + + if (Double.isInfinite(max)) + throw new RuntimeException("Scoreables is probably empty"); + + int i = 0; + for (HasScore scorable : scorables) + res[i++] = Math.exp(scorable.getScore() - max); + NumUtils.normalize(res); + return res; + } + + + + + +} diff --git a/src/edu/stanford/nlp/sempre/ReverseFormula.java b/src/edu/stanford/nlp/sempre/ReverseFormula.java index 25b9aae..3dbc698 100644 --- a/src/edu/stanford/nlp/sempre/ReverseFormula.java +++ b/src/edu/stanford/nlp/sempre/ReverseFormula.java @@ -3,12 +3,14 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * If |expr| denotes a set of pairs S, - * then (reverse |expr|) denotes the set of pairs {(y,x) : (x,y) \in S}. + * then (reverse |expr|) denotes the set of pairs {(y, x) : (x, y) \in S}. * Example: - * (rev fb:people.person.date_of_birth) - * (rev (lambda x (fb:location.statistical_region.population (fb:measurement_unit.dated_integer.number (var x))))) + * (reverse fb:people.person.date_of_birth) + * (reverse (lambda x (fb:location.statistical_region.population (fb:measurement_unit.dated_integer.number (var x))))) * * @author Percy Liang */ @@ -29,6 +31,14 @@ public class ReverseFormula extends Formula { return result == null ? new ReverseFormula(child.map(func)) : result; } + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) + res.addAll(child.mapToList(func, alwaysRecurse)); + return res; + } + @Override public boolean equals(Object thatObj) { if (!(thatObj instanceof ReverseFormula)) return false; @@ -36,7 +46,7 @@ public class ReverseFormula extends Formula { if (!this.child.equals(that.child)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + child.hashCode(); diff --git a/src/edu/stanford/nlp/sempre/Rule.java b/src/edu/stanford/nlp/sempre/Rule.java index c003e4f..12f7ba7 100644 --- a/src/edu/stanford/nlp/sempre/Rule.java +++ b/src/edu/stanford/nlp/sempre/Rule.java @@ -1,7 +1,5 @@ package edu.stanford.nlp.sempre; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.collect.Lists; import fig.basic.LispTree; @@ -26,25 +24,21 @@ public class Rule { public static final String lemmaTokenCat = "$LEMMA_TOKEN"; // Lemmatized version public static final String lemmaPhraseCat = "$LEMMA_PHRASE"; // Lemmatized version - @JsonProperty String lhs; // Left-hand side: category. - @JsonProperty - List rhs; // Right-hand side: sequence of categories (have $ prefix) and tokens. - @JsonProperty - SemanticFn sem; // Takes derivations corresponding to RHS categories and produces a set of derivations corresponding to LHS. - List> info; // Extra info + public final String lhs; // Left-hand side: category. + public final List rhs; // Right-hand side: sequence of categories (have $ prefix) and tokens. + public final SemanticFn sem; // Takes derivations corresponding to RHS categories and produces a set of derivations corresponding to LHS. + public List> info; // Extra info - private String semRepn = null; + // Cache the semanticRepn public String getSemRepn() { if (semRepn == null) semRepn = sem.getClass().getSimpleName(); return semRepn; } + private String semRepn = null; - public Rule() {} - - @JsonCreator - public Rule(@JsonProperty("lhs") String lhs, - @JsonProperty("rhs") List rhs, - @JsonProperty("sem") SemanticFn sem) { + public Rule(String lhs, + List rhs, + SemanticFn sem) { this.lhs = lhs; this.rhs = rhs; this.sem = sem; @@ -58,53 +52,71 @@ public class Rule { } private String stringRepn; // Cache toString() - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Rule rule = (Rule) o; - - if (info != null ? !info.equals(rule.info) : rule.info != null) return false; - if (lhs != null ? !lhs.equals(rule.lhs) : rule.lhs != null) return false; - if (rhs != null ? !rhs.equals(rule.rhs) : rule.rhs != null) return false; - if (sem != null ? !sem.equals(rule.sem) : rule.sem != null) return false; - - return true; - } - - @Override - public int hashCode() { - int hash = 0x7ed55d16; - if(this!=nullRule) { - hash = hash * 0xd3a2646c + lhs.hashCode(); - hash = hash * 0xd3a2646c + rhs.hashCode(); - hash = hash * 0xd3a2646c + sem.toString().hashCode(); - } - return hash; - } - + // Get/set info public void addInfo(String key, double value) { if (info == null) info = Lists.newArrayList(); info.add(Pair.newPair(key, value)); } + public Rule setInfo(Rule rule) { this.info = rule.info; return this; } // Accessors public SemanticFn getSem() { return sem; } + public String getLhs() { return lhs; } // Return whether rule has form A -> B (both LHS and RHS contain one category). public boolean isCatUnary() { return rhs.size() == 1 && isCat(rhs.get(0)); } + // Return if all RHS tokens are terminals + public boolean isRhsTerminals() { + for (int i = 0; i < rhs.size(); ++i) { + if (isCat(rhs.get(i))) + return false; + } + return true; + } + public LispTree toLispTree() { LispTree tree = LispTree.proto.newList(); tree.addChild("rule"); tree.addChild(lhs); tree.addChild(LispTree.proto.newList(rhs)); tree.addChild(sem.toLispTree()); + if (info != null) { + for (Pair p : info) + tree.addChild(LispTree.proto.newList(p.getFirst(), "" + p.getSecond())); + } return tree; } - public String getLhs() { - return lhs; + /* Extract tag info */ + private double getInfoTag(String infoTag) { + if (info != null) { + for (Pair p : info) { + if (p.getFirst().equals(infoTag)) return p.getSecond(); + } + } + return -1.0; + } + + public boolean isFloating() { + double f = getInfoTag("floating"); + double a = getInfoTag("anchored"); + if (f == 1.0) + return true; + else if (f == 0.0) + return false; + else + return a == 1.0 ? false : FloatingParser.opts.defaultIsFloating; + } + + public boolean isAnchored() { + double f = getInfoTag("floating"); + double a = getInfoTag("anchored"); + if (a == 1.0) + return true; + else if (a == 0.0) + return false; + else + return f == 1.0 ? false : !FloatingParser.opts.defaultIsFloating; } } diff --git a/src/edu/stanford/nlp/sempre/SelectFn.java b/src/edu/stanford/nlp/sempre/SelectFn.java index 1f6f898..6f14b99 100644 --- a/src/edu/stanford/nlp/sempre/SelectFn.java +++ b/src/edu/stanford/nlp/sempre/SelectFn.java @@ -4,7 +4,6 @@ import fig.basic.LispTree; import fig.basic.LogInfo; import fig.basic.Option; -import java.util.Collections; import java.util.List; /** @@ -34,40 +33,34 @@ public class SelectFn extends SemanticFn { this.position = Integer.valueOf(tree.child(1).value); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SelectFn selectFn = (SelectFn) o; - if (position != selectFn.position) return false; - return true; - } - - public List call(Example ex, Callable c) { - FeatureVector features = new FeatureVector(); - - // TODO: move into FeatureExtractor - if (FeatureExtractor.containsDomain("skipPos")) { - for (int i = 0; i < c.getChildren().size(); ++i) { - if (i != this.position) { - Derivation child = c.child(i); - for (int index = child.start; index < child.end; ++index) { - List posTags = ex.languageInfo.posTags; - features.add("skipPos", posTags.get(index)); - if (opts.verbose > 0) { - LogInfo.logs( - "SelectFn: adding pos-skipping feature, pos: %s, word: %s", - posTags.get(index), ex.languageInfo.tokens.get(index)); + public DerivationStream call(final Example ex, final Callable c) { + return new SingleDerivationStream() { + @Override + public Derivation createDerivation() { + FeatureVector features = new FeatureVector(); + // TODO(pliang): move into FeatureExtractor + if (FeatureExtractor.containsDomain("skipPos")) { + for (int i = 0; i < c.getChildren().size(); ++i) { + if (i != position) { + Derivation child = c.child(i); + for (int index = child.start; index < child.end; ++index) { + List posTags = ex.languageInfo.posTags; + features.add("skipPos", posTags.get(index)); + if (opts.verbose > 0) { + LogInfo.logs( + "SelectFn: adding pos-skipping feature, pos: %s, word: %s", + posTags.get(index), ex.languageInfo.tokens.get(index)); + } + } } } } + return new Derivation.Builder() + .withCallable(c) + .withFormulaFrom(c.child(position)) + .localFeatureVector(features) + .createDerivation(); } - } - Derivation deriv = new Derivation.Builder() - .withCallable(c) - .withFormulaFrom(c.child(this.position)) - .localFeatureVector(features) - .createDerivation(); - return Collections.singletonList(deriv); + }; } } diff --git a/src/edu/stanford/nlp/sempre/SemType.java b/src/edu/stanford/nlp/sempre/SemType.java index 6d583bd..fc5be94 100644 --- a/src/edu/stanford/nlp/sempre/SemType.java +++ b/src/edu/stanford/nlp/sempre/SemType.java @@ -2,10 +2,10 @@ package edu.stanford.nlp.sempre; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; + import fig.basic.LispTree; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** * A simple type system for Formulas. SemType represents a union over base @@ -18,28 +18,30 @@ import java.util.List; * * Example of a binary (for born in) [remember, arg1 is the argument, arg0 is * the return type] - * (-> fb:location.location * fb:people.person) + * (-> fb:location.location fb:people.person) * - * Note: type equality is not implemented, since it's better to use meet() and - * isSupertypeOf() to exploit the finer lattice structure of the type system. + * Note: type equality is not implemented, since it's better to use meet() to + * exploit the finer lattice structure of the type system. * * @author Percy Liang */ public abstract class SemType { - // Return whether the type is valid. + // Return whether the type is valid (not bottom). public abstract boolean isValid(); // Return the meet of |this| and |that|. public abstract SemType meet(SemType that); - // Return whether |this| is a supertype of |that|. - public abstract boolean isSupertypeOf(SemType that); + // Return the reversed type for functions: (s -> t) to (t -> s) + public abstract SemType reverse(); // Treat |this| as a function type and apply it to the argument |that|. + // This is just an internal function. public abstract SemType apply(SemType that); - // Return the reversed type: (s -> t) to (t -> s) - public abstract SemType reverse(); + // These are really the primitives. + public SemType getArgType() { return reverse().apply(SemType.topType); } + public SemType getRetType() { return apply(SemType.topType); } public abstract LispTree toLispTree(); @@ -53,12 +55,15 @@ public abstract class SemType { } public static SemType fromLispTree(LispTree tree) { - if (tree.isLeaf()) return new EntitySemType(tree.value); + if (tree.isLeaf()) { + if (tree.value.equals("top")) return topType; + return new AtomicSemType(tree.value); + } if ("union".equals(tree.child(0).value)) { - UnionSemType result = new UnionSemType(); + List result = new ArrayList<>(); for (int i = 1; i < tree.children.size(); i++) result.add(fromLispTree(tree.child(i))); - return result; + return new UnionSemType(result); } if ("->".equals(tree.child(0).value)) { SemType result = fromLispTree(tree.child(tree.children.size() - 1)); @@ -69,167 +74,38 @@ public abstract class SemType { throw new RuntimeException("Invalid type: " + tree); } - // Common types + // Create a new instance of SemType from type names (Strings) + + public static SemType newAtomicSemType(String type) { + return new AtomicSemType(type); + } + + public static SemType newFuncSemType(String argType, String retType) { + return new FuncSemType(argType, retType); + } + + public static SemType newUnionSemType(Collection types) { + List t = new ArrayList<>(); + for (String x : types) + t.add(new AtomicSemType(x)); + return new UnionSemType(t).simplify(); + } + + public static SemType newUnionSemType(String... types) { + return newUnionSemType(Arrays.asList(types)); + } + + // Common types + public static final SemType topType = new TopSemType(); public static final SemType bottomType = new UnionSemType(); - public static final SemType stringType = new EntitySemType(FreebaseInfo.TEXT); - public static final SemType numberType = new EntitySemType(FreebaseInfo.NUMBER); - public static final SemType dateType = new EntitySemType(FreebaseInfo.DATE); - public static final SemType entityType = new EntitySemType(FreebaseInfo.ENTITY); - // Everything (ignore cvt and boolean) - public static final SemType topType = UnionSemType.create(stringType, numberType, dateType, entityType); -} - -class EntitySemType extends SemType { - public final String name; - public EntitySemType(String name) { this.name = name; } - public boolean isValid() { return true; } - public SemType meet(SemType that) { - if (that instanceof UnionSemType) return that.meet(this); - if (that instanceof EntitySemType) { - String name1 = this.name; - String name2 = ((EntitySemType) that).name; - //if (!name1.equals(name2)) return SemType.bottomType; - //return this; - - // Check this and that against each other's supertypes. - // Assume a tree-structured lattice. - FreebaseInfo fbInfo = FreebaseInfo.getSingleton(); - if (fbInfo.getIncludedTypesInclusive(name1).contains(name2)) - return this; - if (fbInfo.getIncludedTypesInclusive(name2).contains(name1)) - return that; - return SemType.bottomType; - } - return SemType.bottomType; - } - public boolean isSupertypeOf(SemType that) { - if (that instanceof UnionSemType) { - // Note: it suffices if we're the supertype of one of the baseTypes. - // This is not a conventional definition of union type. - for (SemType baseType : ((UnionSemType) that).baseTypes) - if (isSupertypeOf(baseType)) - return true; - return false; - } - if (that instanceof EntitySemType) { - String name1 = this.name; - String name2 = ((EntitySemType) that).name; - FreebaseInfo fbInfo = FreebaseInfo.getSingleton(); - return fbInfo.getIncludedTypesInclusive(name2).contains(name1); - } - return false; - } - public SemType apply(SemType that) { return SemType.bottomType; } - public SemType reverse() { return SemType.bottomType; } - public LispTree toLispTree() { return LispTree.proto.newLeaf(name); } -} - -class FuncSemType extends SemType { - public final SemType argType; - public final SemType retType; - public FuncSemType(SemType argType, SemType retType) { - this.argType = argType; - this.retType = retType; - } - public boolean isValid() { return true; } - - public SemType meet(SemType that) { return SemType.bottomType; } - - public boolean isSupertypeOf(SemType that) { - if (that instanceof UnionSemType) { - for (SemType baseType : ((UnionSemType) that).baseTypes) - if (isSupertypeOf(baseType)) - return true; - } - if (that instanceof FuncSemType) { - // Return type covaries, arg type contravaries - // Ignore type checking on the argType (contravariance is too strict) - //if (!((FuncSemType)that).argType.isSupertypeOf(argType)) return false; - return retType.isSupertypeOf(((FuncSemType) that).retType); - } - return false; - } - - public SemType apply(SemType that) { - // Currently, we insist that |that| must be a subtype of |argType|. - // This allows us to rule out allowing passing an arbitrary location to - // something expecting a city. - // However, this might be too strong for queries like "lived in something", - // where "something" maps onto fb:common.topic. - // But we assume it is rare in natural language queries to modify with - // something so vague. - //if (argType.meet(baseType).isValid()) return retType; - if (argType.isSupertypeOf(that)) return retType; - return SemType.bottomType; - } - public SemType reverse() { return new FuncSemType(retType, argType); } - public LispTree toLispTree() { - LispTree tree = LispTree.proto.newList(); - tree.addChild("->"); - tree.addChild(argType.toLispTree()); - tree.addChild(retType.toLispTree()); - return tree; - } -} - -class UnionSemType extends SemType { - public final List baseTypes; - public UnionSemType() { this.baseTypes = new ArrayList(); } - public boolean isValid() { return baseTypes.size() > 0; } - - public SemType meet(SemType that) { - UnionSemType result = new UnionSemType(); - for (SemType baseType : baseTypes) - result.add(baseType.meet(that)); - return result.simplify(); - } - - public boolean isSupertypeOf(SemType that) { - // Compute whether there exists a baseType in this that covers (is a supertype of) that. - // TODO: this is technically incorrect because we assume that one baseType can cover |that|. - for (SemType baseType : baseTypes) - if (baseType.isSupertypeOf(that)) - return true; - return false; - } - - public SemType apply(SemType that) { - UnionSemType result = new UnionSemType(); - for (SemType baseType : baseTypes) - result.add(baseType.apply(that)); - return result.simplify(); - } - - public SemType reverse() { - UnionSemType result = new UnionSemType(); - for (SemType baseType : baseTypes) - result.add(baseType.reverse()); - return result.simplify(); - } - - public LispTree toLispTree() { - LispTree result = LispTree.proto.newList(); - result.addChild("union"); - for (SemType baseType : baseTypes) - result.addChild(baseType.toLispTree()); - return result; - } - - public SemType simplify() { - if (baseTypes.size() == 0) return SemType.bottomType; - if (baseTypes.size() == 1) return baseTypes.get(0); - return this; - } - - public UnionSemType add(SemType baseType) { - if (baseType.isValid()) baseTypes.add(baseType); - return this; - } - - public static UnionSemType create(SemType... baseTypes) { - UnionSemType result = new UnionSemType(); - for (SemType baseType : baseTypes) - result.add(baseType); - return result; - } + public static final SemType stringType = new AtomicSemType(CanonicalNames.TEXT); + public static final SemType intType = new AtomicSemType(CanonicalNames.INT); + public static final SemType floatType = new AtomicSemType(CanonicalNames.FLOAT); + public static final SemType dateType = new AtomicSemType(CanonicalNames.DATE); + public static final SemType numberType = new AtomicSemType(CanonicalNames.NUMBER); + public static final SemType numberOrDateType = new UnionSemType(numberType, dateType); + public static final SemType entityType = new AtomicSemType(CanonicalNames.ENTITY); + public static final SemType anyType = new AtomicSemType(CanonicalNames.ANY); + + public static final FuncSemType anyAnyFunc = new FuncSemType(anyType, anyType); } diff --git a/src/edu/stanford/nlp/sempre/SemTypeHierarchy.java b/src/edu/stanford/nlp/sempre/SemTypeHierarchy.java new file mode 100644 index 0000000..3b0d956 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/SemTypeHierarchy.java @@ -0,0 +1,82 @@ +package edu.stanford.nlp.sempre; + +import java.util.*; +import fig.basic.*; + +/** + * Maintain a hierarchy (DAG) over strings. + * subtype < type < supertype + * + * @author Percy Liang + */ +public class SemTypeHierarchy { + public static class Options { + @Option(gloss = "Throw an error if the type is not registered in the type hierarchy.") + public boolean failOnUnknownTypes = false; + } + public static Options opts = new Options(); + + public static final SemTypeHierarchy singleton = new SemTypeHierarchy(); + + // type => list of all supertypes (assume we don't have that many supertypes) + private Map> supertypesMap = new HashMap<>(); // type => supertypes of type + private Map> subtypesMap = new HashMap<>(); // type => subtype of type + // Note: don't always need this, so can maybe remove later + + public SemTypeHierarchy() { + // Add basic types. + addSupertype(CanonicalNames.BOOLEAN, CanonicalNames.BOOLEAN); + addSupertype(CanonicalNames.BOOLEAN, CanonicalNames.ANY); + addSupertype(CanonicalNames.INT, CanonicalNames.INT); + addSupertype(CanonicalNames.INT, CanonicalNames.NUMBER); + addSupertype(CanonicalNames.INT, CanonicalNames.ANY); + addSupertype(CanonicalNames.FLOAT, CanonicalNames.FLOAT); + addSupertype(CanonicalNames.FLOAT, CanonicalNames.NUMBER); + addSupertype(CanonicalNames.FLOAT, CanonicalNames.ANY); + addSupertype(CanonicalNames.DATE, CanonicalNames.DATE); + addSupertype(CanonicalNames.DATE, CanonicalNames.ANY); + addSupertype(CanonicalNames.TEXT, CanonicalNames.TEXT); + addSupertype(CanonicalNames.TEXT, CanonicalNames.ANY); + addSupertype(CanonicalNames.NUMBER, CanonicalNames.NUMBER); + addSupertype(CanonicalNames.NUMBER, CanonicalNames.ANY); + addSupertype(CanonicalNames.ENTITY, CanonicalNames.ENTITY); + addSupertype(CanonicalNames.ENTITY, CanonicalNames.ANY); + addSupertype(CanonicalNames.ANY, CanonicalNames.ANY); + } + + // Add standard supertypes of entity + public void addEntitySupertypes(String type) { + // LogInfo.logs("addEntitySupertypes %s", type); + addSupertype(type, type); + addSupertype(type, CanonicalNames.ENTITY); + addSupertype(type, CanonicalNames.ANY); + } + + // Add: subtype < supertype + public void addSupertype(String subtype, String supertype) { + MapUtils.addToSet(supertypesMap, subtype, supertype); + MapUtils.addToSet(subtypesMap, supertype, subtype); + } + + public Set getSupertypes(String type) { + Set set = supertypesMap.get(type); + if (set == null) { + if (opts.failOnUnknownTypes) + LogInfo.fails("SemTypeHierarchy.getSupertypes: don't know about type %s", type); + addEntitySupertypes(type); + set = supertypesMap.get(type); + } + return set; + } + + public Set getSubtypes(String type) { + Set set = subtypesMap.get(type); + if (set == null) { + if (opts.failOnUnknownTypes) + LogInfo.fails("SemTypeHierarchy.getSubtypes: don't know about type %s", type); + addEntitySupertypes(type); + set = supertypesMap.get(type); + } + return set; + } +} diff --git a/src/edu/stanford/nlp/sempre/SemanticFn.java b/src/edu/stanford/nlp/sempre/SemanticFn.java index 273fb41..e26ad8b 100644 --- a/src/edu/stanford/nlp/sempre/SemanticFn.java +++ b/src/edu/stanford/nlp/sempre/SemanticFn.java @@ -1,17 +1,19 @@ package edu.stanford.nlp.sempre; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; import fig.basic.LispTree; import fig.basic.Option; -import fig.basic.Utils; import java.util.ArrayList; import java.util.List; /** * A semantic function takes a sequence of child derivations and produces a set - * of parent derivations. + * of parent derivations. This is a pretty general concept, which can be used to: + * - Generating candidates (lexicon) + * - Do simple combination + * - Filtering of derivations + * + * To override implement this function, you just need to fill out the call() function. * * @author Percy Liang */ @@ -24,7 +26,7 @@ public abstract class SemanticFn { public static final Options opts = new Options(); - // Used to define this SemanticFn (right now, mostly for printing). + // Used to define this SemanticFn. private LispTree tree; // Initialize the semantic function with any arguments (optional). @@ -33,14 +35,14 @@ public abstract class SemanticFn { this.tree = tree; } - public static interface Callable { - public String getCat(); - public int getStart(); - public int getEnd(); - public Rule getRule(); - public List getChildren(); - public Derivation child(int i); - public String childStringValue(int i); + public interface Callable { + String getCat(); + int getStart(); + int getEnd(); + Rule getRule(); + List getChildren(); + Derivation child(int i); + String childStringValue(int i); } public static class CallInfo implements Callable { @@ -70,33 +72,17 @@ public abstract class SemanticFn { new CallInfo("", -1, -1, Rule.nullRule, new ArrayList()); } - // Main entry point: given information the arguments of the SemanticFn, - // return a list of Derivations. - public abstract List call(Example ex, Callable c); + // Main entry point: return a stream of Derivations (possibly none). + // The computation of the Derivations should be done lazily. + public abstract DerivationStream call(Example ex, Callable c); public LispTree toLispTree() { return tree; } + @Override public String toString() { return tree.toString(); } - public static SemanticFn fromLispTree(LispTree tree) { - String name = tree.child(0).value; - SemanticFn fn = null; + // default does nothing + public void addFeedback(Example ex) { return; } - if (fn == null) - fn = (SemanticFn) Utils.newInstanceHard(Grammar.opts.semanticFnPackage + "." + name); - if (fn == null) - throw new RuntimeException("Invalid SemanticFn name: " + name); + // default does nothing + public void sortOnFeedback(Params params) { return; } - fn.init(tree); - return fn; - } - - @JsonValue - @Override - public String toString() { return tree.toString(); } - - @JsonCreator - public static SemanticFn fromString(String s) { - return fromLispTree(LispTree.proto.parseFromString(s)); - } - - @Override abstract public boolean equals(Object o); } diff --git a/src/edu/stanford/nlp/sempre/SempreUtils.java b/src/edu/stanford/nlp/sempre/SempreUtils.java new file mode 100644 index 0000000..5337e64 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/SempreUtils.java @@ -0,0 +1,34 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LogInfo; +import fig.basic.MapUtils; + +import java.util.Map; + +/** + * Created by joberant on 10/18/14. + */ +public final class SempreUtils { + private SempreUtils() { } + + // "java.util.ArrayList" => "java.util.ArrayList" + // "TypeLookup" => "edu.stanford.nlp.sempre.TypeLookup" + public static String resolveClassName(String name) { + if (name.startsWith("edu.") || name.startsWith("org.") || + name.startsWith("com.") || name.startsWith("net.")) + return name; + return "edu.stanford.nlp.sempre." + name; + } + + public static void logMap(Map map, String desc) { + LogInfo.begin_track("Logging %s map", desc); + for (K key : map.keySet()) + LogInfo.log(key + "\t" + map.get(key)); + LogInfo.end_track(); + } + + public static void addToDoubleMap(Map mutatedMap, Map addedMap) { + for (String key : addedMap.keySet()) + MapUtils.incr(mutatedMap, key, addedMap.get(key)); + } +} diff --git a/src/edu/stanford/nlp/sempre/Server.java b/src/edu/stanford/nlp/sempre/Server.java new file mode 100644 index 0000000..6e2e08a --- /dev/null +++ b/src/edu/stanford/nlp/sempre/Server.java @@ -0,0 +1,678 @@ +package edu.stanford.nlp.sempre; + +import com.google.common.collect.Lists; +import fig.basic.*; +import fig.html.HtmlElement; +import fig.html.HtmlUtils; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URLDecoder; +import java.net.HttpCookie; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.Headers; + +import java.io.*; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; +import java.math.BigInteger; +import java.security.SecureRandom; + +import static fig.basic.LogInfo.logs; + +final class SecureIdentifiers { + private SecureIdentifiers() { } + + private static SecureRandom random = new SecureRandom(); + public static String getId() { + return new BigInteger(130, random).toString(32); + } +} + +/** + * This class implements a simple HTTP server which provides a web interface + * into SEMPRE just like Master.runInteractivePrompt() exposes a command-line + * tool. Most of the work is dispatched to Master.processLine(). + * Cookies are used to store the session ID. + * + * @author Percy Liang + */ +public class Server { + public static class Options { + @Option public int port = 8400; + @Option public int numThreads = 4; + @Option public String title = "SEMPRE Demo"; + @Option public String headerPath; + @Option public String basePath = "demo-www"; + @Option public int verbose = 1; + @Option public int htmlVerbose = 1; + } + public static Options opts = new Options(); + + Master master; + public static final HtmlUtils H = new HtmlUtils(); + + class Handler implements HttpHandler { + public void handle(HttpExchange exchange) { + try { + new ExchangeState(exchange); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + class ExchangeState { + // Input + HttpExchange exchange; + Map reqParams = new HashMap<>(); + String remoteHost; + + // For header + HttpCookie cookie; + boolean isNewSession; + String format; + boolean jsonFormat() { return format.equals("json"); } + + // For writing main content + + public ExchangeState(HttpExchange exchange) throws IOException { + this.exchange = exchange; + + URI uri = exchange.getRequestURI(); + this.remoteHost = exchange.getRemoteAddress().getHostName(); + + // Don't use uri.getQuery: it can't distinguish between '+' and '-' + String[] tokens = uri.toString().split("\\?"); + if (tokens.length == 2) { + for (String s : tokens[1].split("&")) { + String[] kv = s.split("=", 2); + try { + String key = URLDecoder.decode(kv[0], "UTF-8"); + String value = URLDecoder.decode(kv[1], "UTF-8"); + logs("%s => %s", key, value); + reqParams.put(key, value); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + } + this.format = MapUtils.get(reqParams, "format", "html"); + + String cookieStr = exchange.getRequestHeaders().getFirst("Cookie"); + if (cookieStr != null) { // Cookie already exists + cookie = HttpCookie.parse(cookieStr).get(0); + isNewSession = false; + } else { + if (!jsonFormat()) { + cookie = new HttpCookie("sessionId", SecureIdentifiers.getId()); + } else { + cookie = null; + } + isNewSession = true; // Create a new cookie + } + + String sessionId = null; + if (cookie != null) sessionId = cookie.getValue(); + if (opts.verbose >= 2) + LogInfo.logs("GET %s from %s (%ssessionId=%s)", uri, remoteHost, isNewSession ? "new " : "", sessionId); + + String uriPath = uri.getPath(); + if (uriPath.equals("/")) uriPath += "index.html"; + if (uriPath.equals("/sempre")) { + handleQuery(sessionId); + } else { + getFile(opts.basePath + uriPath); + } + + exchange.close(); + } + + String getMimeType(String path) { + String[] tokens = path.split("\\."); + String ext = tokens[tokens.length - 1]; + if (ext.equals("html")) return "text/html"; + if (ext.equals("css")) return "text/css"; + if (ext.equals("jpeg")) return "image/jpeg"; + if (ext.equals("gif")) return "image/gif"; + return "text/plain"; + } + + void setHeaders(String mimeType) throws IOException { + Headers headers = exchange.getResponseHeaders(); + headers.set("Content-Type", mimeType); + headers.set("Access-Control-Allow-Origin", "*"); + if (isNewSession && cookie != null) + headers.set("Set-Cookie", cookie.toString()); + exchange.sendResponseHeaders(200, 0); + } + + private HtmlElement makeInputBox(String line, String action) { + return H.div().child( + H.form().action(action) + .child(H.text(line == null ? "" : line).cls("question").autofocus().size(50).name("q")) + .child(H.button("Go").cls("ask")) + .end()); + } + + private HtmlElement makeTooltip(HtmlElement main, HtmlElement aux) { + return H.a().cls("info").child(main).child(H.span().cls("tooltip").child(aux)); + } + private HtmlElement makeTooltip(HtmlElement main, HtmlElement aux, String link) { + return H.a().href(link).cls("info").child(main).child(H.span().cls("tooltip").child(aux)); + } + + public final String freebaseWebsite = "http://www.freebase.com/"; + public String id2website(String id) { + assert id.startsWith("fb:") : id; + return freebaseWebsite + id.substring(3).replaceAll("\\.", "/"); + } + + HtmlElement valueToElem(Value value) { + if (value == null) return H.span(); + if (value instanceof NameValue) { + NameValue nameValue = (NameValue) value; + return H.a().href(id2website(nameValue.id)).child(nameValue.description == null ? nameValue.id : nameValue.description); + } else if (value instanceof NumberValue) { + NumberValue numberValue = (NumberValue) value; + return H.span().child(Fmt.D(numberValue.value) + (numberValue.unit.equals(NumberValue.unitless) ? "" : " " + numberValue.unit)); + } else if (value instanceof UriValue) { + UriValue uriValue = (UriValue) value; + return H.a().href(uriValue.value).child(uriValue.value); + } else if (value instanceof DateValue) { + DateValue dateValue = (DateValue) value; + return H.span().child(dateValue.year + (dateValue.month == -1 ? "" : "-" + dateValue.month + (dateValue.day == -1 ? "" : "-" + dateValue.day))); + } else if (value instanceof StringValue) { + return H.span().child(((StringValue) value).value); + } else if (value instanceof TableValue) { + HtmlElement table = H.table().cls("valueTable"); + HtmlElement header = H.tr(); + boolean first = true; + for (String item : ((TableValue) value).header) { + if (!first) header.child(H.td("   ")); + first = false; + header.child(H.td(H.b(item))); + } + table.child(header); + for (List rowValues : ((TableValue) value).rows) { + HtmlElement row = H.tr(); + first = true; + for (Value x : rowValues) { + // TODO(pliang): add horizontal spacing only using CSS + if (!first) row.child(H.td("   ")); + first = false; + row.child(H.td(valueToElem(x))); + } + table.child(row); + } + return table; + } else { + // Default rendering + return H.span().child(value.toString()); + } + } + + private HtmlElement makeAnswerBox(Master.Response response, String uri) { + HtmlElement answer; + if (response.getExample().getPredDerivations().size() == 0) { + answer = H.span().child("(none)"); + } else { + answer = valueToElem(response.getDerivation().getValue()); + } + + return H.table().child( + H.tr() + .child( + H.td( + makeTooltip( + H.span().cls("correctButton").child("[Correct]"), + H.div().cls("bubble").child("If this answer is correct, click to add as a new training example!"), + uri + "&accept=" + response.getCandidateIndex()))) + .child(H.td(H.span().cls("answer").child(answer))) + .end()); + } + + private HtmlElement makeGroup(List items) { + HtmlElement table = H.table().cls("groupResponse"); + for (HtmlElement item : items) + table.child(H.tr().child(H.td(item))); + return table; + } + + HtmlElement makeDetails(Master.Response response, String uri) { + Example ex = response.getExample(); + List items = new ArrayList(); + if (opts.htmlVerbose >= 1) + items.add(makeLexical(ex)); + if (ex.getPredDerivations().size() > 0) { + if (opts.htmlVerbose >= 1) { + items.add(makeDerivation(ex, response.getDerivation(), true)); + items.add(makeFeatures(response.getDerivation(), false)); + } + items.add(makeCandidates(ex, uri)); + } + + return H.div().cls("details").child(makeGroup(items)); + } + + HtmlElement makeDerivation(Example ex, Derivation deriv, boolean moreInfo) { + HtmlElement table = H.table(); + + // Show the derivation + table.child(H.tr().child(H.td(makeDerivationHelper(ex, deriv, "", moreInfo)))); + + String header = "Derivation"; + return H.div() + .child(H.span().cls("listHeader").child(header)) + .child(table); + } + + HtmlElement makeDerivationHelper(Example ex, Derivation deriv, String indent, boolean moreInfo) { + // TODO(pliang): make this prettier + HtmlElement cat; + if (moreInfo) { + HtmlElement tooltip = H.div(); + tooltip.child(H.span(deriv.rule.toString()).cls("nowrap")); + tooltip.child(makeFeatures(deriv, true)); + cat = makeTooltip(H.span(deriv.cat), tooltip); + } else { + cat = H.span(deriv.cat); + } + String description = cat + "[ " + H.span().child(ex.phraseString(deriv.start, deriv.end)).cls("word") + "]" + " → " + deriv.formula; + HtmlElement node = H.div().child(indent + description); + for (Derivation child : deriv.children) + node.child(makeDerivationHelper(ex, child, indent + "    ", moreInfo)); + return node; + } + + HtmlElement makeFeatures(Derivation deriv, boolean local) { + HtmlElement table = H.table(); + + Params params = master.getParams(); + Map features = new HashMap(); + if (local) + deriv.incrementLocalFeatureVector(1, features); + else + deriv.incrementAllFeatureVector(1, features); + + List> entries = Lists.newArrayList(); + double sumValue = 0; + for (Map.Entry entry : features.entrySet()) { + String feature = entry.getKey(); + if (entry.getValue() == 0) continue; + double value = entry.getValue() * params.getWeight(feature); + sumValue += value; + entries.add(new java.util.AbstractMap.SimpleEntry(feature, value)); + } + Collections.sort(entries, new ValueComparator(false)); + table.child( + H.tr() + .child(H.td(H.b("Feature"))) + .child(H.td(H.b("Value"))) + .child(H.td(H.b("Weight")))); + + for (Map.Entry entry : entries) { + String feature = entry.getKey(); + double value = entry.getValue(); + double weight = params.getWeight(feature); + table.child( + H.tr() + .child(H.td(feature)) + .child(H.td(Fmt.D(MapUtils.getDouble(features, feature, 0)))) + .child(H.td(Fmt.D(weight)))); + } + + String header; + if (local) { + double localScore = deriv.localScore(params); + double score = deriv.getScore(); + if (deriv.children == null) + header = String.format("Local features (score = %s)", Fmt.D(score)); + else + header = String.format("Local features (score = %s + %s = %s)", Fmt.D(score - localScore), Fmt.D(localScore), Fmt.D(score)); + } else { + header = String.format("All features (score=%s, prob=%s)", Fmt.D(deriv.getScore()), Fmt.D(deriv.getProb())); + } + return H.div() + .child(H.span().cls("listHeader").child(header)) + .child(table); + } + + HtmlElement linkSelect(int index, String uri, String str) { + return H.a().href(uri + "&select=" + index).child(str); + } + + private HtmlElement makeCandidates(Example ex, String uri) { + HtmlElement table = H.table().cls("candidateTable"); + HtmlElement header = H.tr() + .child(H.td(H.b("Rank"))) + .child(H.td(H.b("Score"))) + .child(H.td(H.b("Answer"))); + if (opts.htmlVerbose >= 1) + header.child(H.td(H.b("Formula"))); + table.child(header); + for (int i = 0; i < ex.getPredDerivations().size(); i++) { + Derivation deriv = ex.getPredDerivations().get(i); + + HtmlElement correct = makeTooltip( + H.span().cls("correctButton").child("[Correct]"), + H.div().cls("bubble").child("If this answer is correct, click to add as a new training example!"), + uri + "&accept=" + i); + String value = shorten(deriv.getValue() == null ? "" : deriv.getValue().toString(), 200); + HtmlElement formula = makeTooltip( + H.span(deriv.getFormula().toString()), + H.div().cls("nowrap").child(makeDerivation(ex, deriv, false)), + uri + "&select=" + i); + HtmlElement row = H.tr() + .child(H.td(linkSelect(i, uri, i + " " + correct)).cls("nowrap")) + .child(H.td(Fmt.D(deriv.getScore()))) + .child(H.td(value)).style("width:250px"); + if (opts.htmlVerbose >= 1) + row.child(H.td(formula)); + table.child(row); + } + return H.div() + .child(H.span().cls("listHeader").child("Candidates")) + .child(table); + } + + private String shorten(String s, int n) { + if (s.length() <= n) return s; + return s.substring(0, n / 2) + "..." + s.substring(s.length() - n / 2); + } + + private void markLexical(Derivation deriv, CandidatePredicates[] predicates) { + // TODO(pliang): generalize this to the case where the formula is a + // NameFormula but the child is a StringFormula? + if (deriv.getRule() != null && + deriv.getRule().getSem() != null && + deriv.getRule().getSem().getClass().getSimpleName().equals("LexiconFn")) + predicates[deriv.getStart()].add(deriv.getFormula(), deriv.getEnd() - deriv.getStart(), deriv.getScore()); + for (Derivation child : deriv.getChildren()) + markLexical(child, predicates); + } + + class CandidatePredicates { + // Parallel arrays + List predicates = new ArrayList(); + List spanLengths = new ArrayList(); + List scores = new ArrayList(); + + void add(Formula formula, int spanLength, double score) { + predicates.add(formula); + spanLengths.add(spanLength); + scores.add(score); + } + int size() { return predicates.size(); } + + String formatPredicate(int i) { + return predicates.get(i).toString() + (spanLengths.get(i) == 1 ? "" : " [" + spanLengths.get(i) + "]"); + } + } + + // Move to fig + double[] toDoubleArray(List l) { + double[] a = new double[l.size()]; + for (int i = 0; i < l.size(); i++) a[i] = l.get(i); + return a; + } + + + HtmlElement makeLexical(Example ex) { + HtmlElement predicatesElem = H.tr(); + HtmlElement tokensElem = H.tr(); + + // Mark all the predicates used in any derivation on the beam. + // Note: this is not all possible. + CandidatePredicates[] predicates = new CandidatePredicates[ex.getTokens().size()]; + for (int i = 0; i < ex.getTokens().size(); i++) + predicates[i] = new CandidatePredicates(); + for (Derivation deriv : ex.getPredDerivations()) + markLexical(deriv, predicates); + + // Build up |predicatesElem| and |tokensElem| + for (int i = 0; i < ex.getTokens().size(); i++) { + tokensElem.child( + H.td( + makeTooltip( + H.span().cls("word").child(ex.getTokens().get(i)), + H.span().cls("tag").child("POS: " + ex.languageInfo.posTags.get(i)), + ""))); + + if (predicates[i].size() == 0) { + predicatesElem.child(H.td("")); + } else { + // Show possible predicates for a word + HtmlElement pe = H.table().cls("predInfo"); + int[] perm = ListUtils.sortedIndices(toDoubleArray(predicates[i].scores), true); + Set formulaSet = new HashSet(); + for (int j : perm) { + String formula = predicates[i].formatPredicate(j); + if (formulaSet.contains(formula)) continue; // Dedup + formulaSet.add(formula); + double score = predicates[i].scores.get(j); + pe.child(H.tr().child(H.td(formula)).child(H.td(Fmt.D(score)))); + } + predicatesElem.child(H.td(makeTooltip(H.span().child(predicates[i].formatPredicate(perm[0])), pe, ""))); + } + } + + return H.div().cls("lexicalResponse") + .child(H.span().cls("listHeader").child("Lexical Triggers")) + .child(H.table().child(predicatesElem).child(tokensElem)); + } + + String makeJson(Master.Response response) { + Map json = new HashMap(); + List items = new ArrayList(); + json.put("candidates", items); + for (Derivation deriv : response.getExample().getPredDerivations()) { + Map item = new HashMap(); + Value value = deriv.getValue(); + if (value instanceof UriValue) { + item.put("url", ((UriValue) value).value); + } else if (value instanceof TableValue) { + TableValue tableValue = (TableValue) value; + item.put("header", tableValue.header); + List> rowsObj = new ArrayList>(); + item.put("rows", rowsObj); + for (List row : tableValue.rows) { + List rowObj = new ArrayList(); + for (Value v : row) + rowObj.add(v.toString()); + rowsObj.add(rowObj); + } + } else { + item.put("value", value.toString()); + } + item.put("score", deriv.score); + item.put("prob", deriv.prob); + items.add(item); + } + + return Json.writeValueAsStringHard(json); + } + + // Catch exception if any. + Master.Response processQuery(Session session, String query) { + try { + return master.processQuery(session, query); + } catch (Throwable t) { + t.printStackTrace(); + return null; + } + } + + // If query is not already the last query, make it the last query. + boolean ensureQueryIsLast(Session session, String query) { + if (query != null && !query.equals(session.getLastQuery())) { + Master.Response response = processQuery(session, query); + if (response == null) return false; + } + return true; + } + + void handleQuery(String sessionId) throws IOException { + String query = reqParams.get("q"); + + // If JSON, don't store cookies. + Session session = master.getSession(sessionId); + session.remoteHost = remoteHost; + session.format = format; + + if (query == null) query = session.getLastQuery(); + if (query == null) query = ""; + logs("Server.handleQuery %s: %s", session.id, query); + + // Print header + if (jsonFormat()) + setHeaders("application/json"); + else + setHeaders("text/html"); + PrintWriter out = new PrintWriter(new OutputStreamWriter(exchange.getResponseBody())); + if (!jsonFormat()) { + out.println(""); + out.println(H.html().open()); + out.println( + H.head() + .child(H.title(opts.title)) + .child(H.link().rel("stylesheet").type("text/css").href("main.css")) + .child(H.script().src("main.js")) + .end()); + + out.println(H.body().open()); + + if (opts.headerPath != null) { + for (String line : IOUtils.readLinesHard(opts.headerPath)) + out.println(line); + } + } + + String uri = exchange.getRequestURI().toString(); + + // Encode the URL parameters into the freeform text. + // A bit backwards, but keeps uniformity. + String select = reqParams.get("select"); + if (select != null) { + if (ensureQueryIsLast(session, query)) + query = LispTree.proto.newList("select", select).toString(); + else + query = null; + } + String accept = reqParams.get("accept"); + if (accept != null) { + if (ensureQueryIsLast(session, query)) + query = LispTree.proto.newList("accept", accept).toString(); + else + query = null; + } + + // Handle the request + Master.Response masterResponse = null; + if (query != null) + masterResponse = processQuery(session, query); + + // Print history of exchanges + if (session.context.exchanges.size() > 0 && !jsonFormat()) { + HtmlElement context = H.table().cls("context"); + for (ContextValue.Exchange e : session.context.exchanges) { + HtmlElement row = H.tr().child(H.td(H.span().cls("word").child(e.utterance))); + row.child(H.td(H.span("    "))).child(H.td(e.value.toString())); + if (opts.htmlVerbose >= 1) + row.child(H.td(H.span("    "))).child(H.td(e.formula.toString())); + context.child(row); + } + out.println(context.toString()); + } + + // Print input box for new utterance + if (!jsonFormat()) { + String defaultQuery = query != null ? query : session.getLastQuery(); + out.println(makeInputBox(defaultQuery, uri).toString()); + } + + if (masterResponse != null) { + // Render answer + Example ex = masterResponse.getExample(); + if (ex != null) { + if (!jsonFormat()) { + out.println(makeAnswerBox(masterResponse, uri).toString()); + out.println(makeDetails(masterResponse, uri).toString()); + } else { + out.println(makeJson(masterResponse)); + } + } + + if (!jsonFormat() && opts.htmlVerbose >= 1) { + // Write response to user + out.println(H.elem("pre").open()); + for (String outLine : masterResponse.getLines()) + out.println(outLine); + out.println(H.elem("pre").close()); + } + } else { + if (query != null && !jsonFormat()) + out.println(H.span("Internal error!").cls("error")); + } + + if (!jsonFormat()) { + out.println(H.body().close()); + out.println(H.html().close()); + } + + out.close(); + } + + void getResults() throws IOException { + setHeaders("application/json"); + Map map = new HashMap<>(); + map.put("a", "3"); + map.put("b", "4"); + + PrintWriter writer = new PrintWriter(new OutputStreamWriter(exchange.getResponseBody())); + writer.println(Json.writeValueAsStringHard(map)); + writer.close(); + } + + void getFile(String path) throws IOException { + if (!new File(path).exists()) { + LogInfo.logs("File doesn't exist: %s", path); + exchange.sendResponseHeaders(404, 0); // File not found + return; + } + + setHeaders(getMimeType(path)); + if (opts.verbose >= 2) + LogInfo.logs("Sending %s", path); + OutputStream out = new BufferedOutputStream(exchange.getResponseBody()); + InputStream in = new FileInputStream(path); + IOUtils.copy(in, out); + } + } + + public Server(Master master) { + this.master = master; + } + + void run() { + try { + String hostname = fig.basic.SysInfoUtils.getHostName(); + HttpServer server = HttpServer.create(new InetSocketAddress(opts.port), 10); + ExecutorService pool = Executors.newFixedThreadPool(opts.numThreads); + server.createContext("/", new Handler()); + server.setExecutor(pool); + server.start(); + LogInfo.logs("Server started at http://%s:%s/sempre", hostname, opts.port); + LogInfo.log("Press Ctrl-D to terminate."); + while (LogInfo.stdin.readLine() != null) { } + LogInfo.log("Shutting down server..."); + server.stop(0); + LogInfo.log("Shutting down executor pool..."); + pool.shutdown(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/edu/stanford/nlp/sempre/Session.java b/src/edu/stanford/nlp/sempre/Session.java index 9bb055b..56aaf93 100644 --- a/src/edu/stanford/nlp/sempre/Session.java +++ b/src/edu/stanford/nlp/sempre/Session.java @@ -2,32 +2,65 @@ package edu.stanford.nlp.sempre; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.HashMap; -import fig.basic.LispTree; -/** A Session contains the information specific to an individual. */ +/** + * A Session contains the information specific to a user. + * It maintains the context for discourse as well as the last example, so that + * we can inspect the different predicted derivations and generate new training + * examples / update parameters interactively. + * + * @author Percy Liang + */ public class Session { - String id; // Session id - String lastQuery; // Last thing query (non-command) typed by the user. - String lastRemoteAddr; - - Map macros = new HashMap(); // Map from macro name to its replacement value - - List examples = new ArrayList(); // List of recent examples. + public final String id; // Session id + String remoteHost; // Where we connected from + String format; // html or json + ContextValue context; // Current context used to create new examples + Example lastEx; // Last example that we processed public Session(String id) { this.id = id; + context = new ContextValue(id, DateValue.now(), new ArrayList()); } - public void setLastRemoteAddr(String lastRemoteAddr) { - this.lastRemoteAddr = lastRemoteAddr; + public Example getLastExample() { return lastEx; } + public String getLastQuery() { return lastEx == null ? null : lastEx.utterance; } + + public void updateContext() { + context = context.withDate(DateValue.now()); } - public String getLastQuery() { return lastQuery; } + public void updateContext(Example ex, int maxExchanges) { + lastEx = ex; + List derivations = lastEx.getPredDerivations(); + if (derivations.size() > 0) { + Derivation deriv = derivations.get(0); + List newExchanges = new ArrayList(); + newExchanges.addAll(context.exchanges); + newExchanges.add(new ContextValue.Exchange(ex.utterance, deriv.formula, deriv.value)); + while (newExchanges.size() > maxExchanges) + newExchanges.remove(0); + context = context.withNewExchange(newExchanges); + } + } + + public void updateContextWithNewAnswer(Example ex, Derivation deriv) { + List newExchanges = new ArrayList(); + for (int i = 0; i < context.exchanges.size() - 1; i++) + newExchanges.add(context.exchanges.get(i)); + newExchanges.add(new ContextValue.Exchange(ex.utterance, deriv.formula, deriv.value)); + context = context.withNewExchange(newExchanges); + } + + public ContextValue getContextExcludingLast() { + List newExchanges = new ArrayList(); + for (int i = 0; i < context.exchanges.size() - 1; i++) + newExchanges.add(context.exchanges.get(i)); + return context.withNewExchange(newExchanges); + } @Override public String toString() { - return String.format("%s [%d examples, %d macros]; last: %s", id, examples.size(), macros.size(), lastQuery); + return String.format("%s: %s; last: %s", id, context, lastEx); } } diff --git a/src/edu/stanford/nlp/sempre/SimpleAnalyzer.java b/src/edu/stanford/nlp/sempre/SimpleAnalyzer.java new file mode 100644 index 0000000..86a7a17 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/SimpleAnalyzer.java @@ -0,0 +1,86 @@ +package edu.stanford.nlp.sempre; + +import java.util.Arrays; + +/** + * SimpleAnalyzer takes an utterance and applies simple methods to pre-process + * + * @author akchou + */ +public class SimpleAnalyzer extends LanguageAnalyzer { + + // Stanford tokenizer doesn't break hyphens. + // Replace hypens with spaces for utterances like + // "Spanish-speaking countries" but not for "2012-03-28". + public static String breakHyphens(String utterance) { + StringBuilder buf = new StringBuilder(utterance); + for (int i = 0; i < buf.length(); i++) { + if (buf.charAt(i) == '-' && (i + 1 < buf.length() && Character.isLetter(buf.charAt(i + 1)))) + buf.setCharAt(i, ' '); + } + return buf.toString(); + } + + private static final String[] numbers = {"zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"}; + + public LanguageInfo analyze(String utterance) { + LanguageInfo languageInfo = new LanguageInfo(); + + // Clear these so that analyze can hypothetically be called + // multiple times. + languageInfo.tokens.clear(); + languageInfo.posTags.clear(); + languageInfo.nerTags.clear(); + languageInfo.nerValues.clear(); + languageInfo.lemmaTokens.clear(); + + // Break hyphens + utterance = breakHyphens(utterance); + + // Default analysis - create tokens crudely + utterance = utterance.replaceAll(",", " ,"); + utterance = utterance.replaceAll("\\.", " ."); + utterance = utterance.replaceAll("\\?", " ?"); + utterance = utterance.replaceAll("'", " '"); + utterance = utterance.replaceAll("\\[", " [ "); + utterance = utterance.replaceAll("\\]", " ] "); + utterance = utterance.trim(); + if (!utterance.equals("")) { + for (String token : utterance.split("\\s+")) { + languageInfo.tokens.add(LanguageAnalyzer.opts.lowerCaseTokens ? token.toLowerCase() : token); + String lemma = token; + if (token.endsWith("s")) + lemma = token.substring(0, token.length() - 1); + languageInfo.lemmaTokens.add(LanguageAnalyzer.opts.lowerCaseTokens ? lemma.toLowerCase() : lemma); + + // Is it a written out number? + int x = Arrays.asList(numbers).indexOf(token); + if (x != -1) { + languageInfo.posTags.add("CD"); + languageInfo.nerTags.add("NUMBER"); + languageInfo.nerValues.add(x + ""); + continue; + } + + try { + Double.parseDouble(token); + languageInfo.posTags.add("CD"); + if (token.length() == 4) + languageInfo.nerTags.add("DATE"); + else + languageInfo.nerTags.add("NUMBER"); + languageInfo.nerValues.add(token); + } catch (NumberFormatException e) { + // Guess that capitalized nouns are proper + if (Character.isUpperCase(token.charAt(0))) + languageInfo.posTags.add("NNP"); + else + languageInfo.posTags.add("UNK"); + languageInfo.nerTags.add("UNK"); + languageInfo.nerValues.add("UNK"); + } + } + } + return languageInfo; + } +} diff --git a/src/edu/stanford/nlp/sempre/SimpleLexicon.java b/src/edu/stanford/nlp/sempre/SimpleLexicon.java new file mode 100644 index 0000000..59f3d93 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/SimpleLexicon.java @@ -0,0 +1,123 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.*; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.*; + +/** + * A Lexicon maps phrases (e.g., born) to lexical entries, which contain a + * formula (e.g., fb:people.person.place_of_birth) and a type. + * This class is meant to be simpler/faster than the normal + * UnaryLexicon/BinaryLexicon. + * + * Note: this class exists because it was annoying to add types into the normal + * UnaryLexicon and I didn't want to break backward compatibility with the rest + * of the code. Also, the normal lexicon is really slow with all those hash + * maps. + * + * The lexicon here only knows about Formulas and fuzzy matching. + * In the lexicon, we have rawPhrase. + * In the user query, we have phrase. + * + * @author Percy Liang + */ +public final class SimpleLexicon { + public static class Entry { + // rawPhrase was the original phrase in the Lexicon + public Entry(String rawPhrase, Formula formula, SemType type, StringDoubleVec features) { + this.rawPhrase = rawPhrase; + this.formula = formula; + this.type = type; + this.features = features; + } + public final String rawPhrase; + public final Formula formula; + public final SemType type; + public final StringDoubleVec features; + + @Override public String toString() { + return "[" + rawPhrase + " => " + formula + " : " + type + "]"; + } + } + + public static class Options { + @Option(gloss = "Path to load lexicon files from") public List inPaths; + @Option(gloss = "Types to allow suffix (last word) matche (for people names") public List matchSuffixTypes; + } + public static Options opts = new Options(); + + private static SimpleLexicon lexicon; + public static SimpleLexicon getSingleton() { + if (lexicon == null) lexicon = new SimpleLexicon(); + return lexicon; + } + + private SimpleLexicon() { + if (opts.inPaths == null) return; + for (String path : opts.inPaths) read(path); + } + + // Mapping from phrase + Map> entries = new HashMap>(); + + public void read(String path) { + LogInfo.begin_track("SimpleLexicon.read(%s)", path); + try { + BufferedReader in = IOUtils.openIn(path); + String line; + int numLines = 0; + int oldNumEntries = entries.size(); + while ((line = in.readLine()) != null) { + Map map = Json.readMapHard(line); + numLines++; + + String rawPhrase = (String) map.get("lexeme"); + Formula formula = Formula.fromString((String) map.get("formula")); + + // Type + String typeStr = (String) map.get("type"); + SemType type = typeStr != null ? SemType.fromString(typeStr) : TypeInference.inferType(formula); + + // Features + StringDoubleVec features = null; + Map featureMap = (Map) map.get("features"); + if (featureMap != null) { + features = new StringDoubleVec(); + for (Map.Entry e : featureMap.entrySet()) + features.add(e.getKey(), e.getValue()); + features.trimToSize(); + } + + // Add verbatim feature + Entry entry = new Entry(rawPhrase, formula, type, features); + String phrase = entry.rawPhrase.toLowerCase(); + MapUtils.addToList(entries, phrase, entry); + + // For last names + String[] parts = phrase.split(" "); + if (opts.matchSuffixTypes != null && opts.matchSuffixTypes.contains(typeStr) && parts.length > 1) { + StringDoubleVec newFeatures = new StringDoubleVec(); + if (features != null) { // Copy over features + for (StringDoubleVec.Entry e : features) + newFeatures.add(e.getFirst(), e.getSecond()); + } + newFeatures.add("isSuffix", 1); + newFeatures.trimToSize(); + Entry newEntry = new Entry(rawPhrase, formula, type, newFeatures); + MapUtils.addToList(entries, parts[parts.length - 1], newEntry); + } + // In the future, add other mechanisms for lemmatization. + } + LogInfo.logs("Read %s lines, generated %d entries (now %d total)", numLines, entries.size() - oldNumEntries, entries.size()); + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.end_track(); + } + + public List lookup(String phrase) { + return MapUtils.get(entries, phrase, Collections.EMPTY_LIST); + } +} diff --git a/src/edu/stanford/nlp/sempre/SimpleLexiconFn.java b/src/edu/stanford/nlp/sempre/SimpleLexiconFn.java new file mode 100644 index 0000000..05223bc --- /dev/null +++ b/src/edu/stanford/nlp/sempre/SimpleLexiconFn.java @@ -0,0 +1,107 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.*; +import java.util.*; + +/** + * Uses the SimpleLexicon. + * + * Example: + * (rule $ROOT ($PHRASE) (SimpleLexiconFn (type fb:type.any))) + * + * @author Percy Liang + */ +public class SimpleLexiconFn extends SemanticFn { + public static class Options { + @Option(gloss = "Number of entities to return from entity lexicon") + public int maxEntityEntries = 100; + + @Option(gloss = "Verbosity level") public int verbose = 0; + } + + public static Options opts = new Options(); + + private static SimpleLexicon lexicon; + + // Only return entries whose type matches this + private SemType restrictType = SemType.topType; + + public SimpleLexiconFn() { + lexicon = SimpleLexicon.getSingleton(); + } + + public void init(LispTree tree) { + super.init(tree); + for (int i = 1; i < tree.children.size(); i++) { + // (type fb:people.person): allow us to restrict the type + LispTree arg = tree.child(i); + if ("type".equals(arg.child(0).value)) { + restrictType = SemType.fromLispTree(arg.child(1)); + } + } + } + + public DerivationStream call(Example ex, Callable c) { + String phrase = c.childStringValue(0); + List entries = lexicon.lookup(phrase); + + // Filter by type + List newEntries = new ArrayList(); + for (SimpleLexicon.Entry e : entries) { + if (opts.verbose >= 3) + LogInfo.logs("SimpleLexiconFn: %s => %s [type = %s meet-> %s]", phrase, e.formula, e.type, restrictType.meet(e.type)); + if (!restrictType.meet(e.type).isValid()) continue; + newEntries.add(e); + } + entries = newEntries; + + return new MyDerivationStream(ex, c, entries, phrase); + } + + public class MyDerivationStream extends MultipleDerivationStream { + private Example ex; + private Callable callable; + private List entries; + private String phrase; + private int currIndex = 0; + + public MyDerivationStream(Example ex, Callable c, List entries, String phrase) { + this.ex = ex; + this.callable = c; + this.entries = entries; + this.phrase = phrase; + } + + @Override public int estimatedSize() { return entries.size(); } + + @Override + public Derivation createDerivation() { + if (currIndex == entries.size()) return null; + + SimpleLexicon.Entry entry = entries.get(currIndex++); + FeatureVector features = new FeatureVector(); + Derivation deriv = new Derivation.Builder() + .withCallable(callable) + .formula(entry.formula) + .type(entry.type) + .localFeatureVector(features) + .createDerivation(); + + if (FeatureExtractor.containsDomain("basicStats")) { + if (entry.features != null) { + for (StringDoubleVec.Entry e : entry.features) + features.add("basicStats", e.getFirst(), e.getSecond()); + } + } + + // Doesn't generalize, but add it for now, otherwise not separable + if (FeatureExtractor.containsDomain("lexAlign")) + deriv.addFeature("lexAlign", phrase + " --- " + entry.formula); + + if (SemanticFn.opts.trackLocalChoices) + deriv.addLocalChoice("SimpleLexiconFn " + deriv.startEndString(ex.getTokens()) + " " + entry); + + return deriv; + } + } +} diff --git a/src/edu/stanford/nlp/sempre/SingleDerivationStream.java b/src/edu/stanford/nlp/sempre/SingleDerivationStream.java new file mode 100644 index 0000000..92bf3ee --- /dev/null +++ b/src/edu/stanford/nlp/sempre/SingleDerivationStream.java @@ -0,0 +1,43 @@ +package edu.stanford.nlp.sempre; + +/** + * Encapsulates the production of at most one Derivation. + */ +public abstract class SingleDerivationStream implements DerivationStream { + private Derivation nextDeriv; // Next one to return. + private boolean consumed; + + // Override this class: should create a new Derivation. + // Return null if there is none. + public abstract Derivation createDerivation(); + + @Override public boolean hasNext() { + if (nextDeriv != null) return true; // Still one in the queue + if (consumed) return false; // No more + nextDeriv = createDerivation(); // Ask for one + consumed = true; + return nextDeriv != null; + } + + @Override public Derivation peek() { + if (nextDeriv == null) throw new RuntimeException("No more derivations!"); + return nextDeriv; + } + + @Override public Derivation next() { + if (!hasNext()) throw new RuntimeException("No more derivations!"); + Derivation deriv = nextDeriv; + nextDeriv = null; + return deriv; + } + + @Override public void remove() { throw new RuntimeException("Cannot remove from DerivationStream"); } + + @Override public int estimatedSize() { return 1; } + + public static SingleDerivationStream constant(final Derivation deriv) { + return new SingleDerivationStream() { + public Derivation createDerivation() { return deriv; } + }; + } +} diff --git a/src/edu/stanford/nlp/sempre/StringCache.java b/src/edu/stanford/nlp/sempre/StringCache.java deleted file mode 100644 index 2f08776..0000000 --- a/src/edu/stanford/nlp/sempre/StringCache.java +++ /dev/null @@ -1,136 +0,0 @@ -package edu.stanford.nlp.sempre; - -import fig.basic.IOUtils; -import fig.basic.LogInfo; - -import java.io.*; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; -import java.util.LinkedHashMap; - -/** - * Stores a Map, which is synchronized with disk. - * - * @author Percy Liang - */ -public interface StringCache { - public String get(String key); - public void put(String key, String value); -} - -/** - * Cache backed by a file. - * - * @author Percy Liang - */ -class FileStringCache implements StringCache { - String path; - LinkedHashMap cache = new LinkedHashMap(); - PrintWriter out; - - public FileStringCache(String path) { - if (path == null) return; - - this.path = path; - - // Read existing. - if (new File(path).exists()) { - try { - BufferedReader in = IOUtils.openInHard(path); - String line; - while ((line = in.readLine()) != null) { - String[] tokens = line.split("\t", 2); - if (tokens.length != 2) - throw new RuntimeException("Invalid line in cache file: " + line); - cache.put(tokens[0], tokens[1]); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - out = IOUtils.openOutAppendHard(path); - LogInfo.logs("Using cache %s (%d entries)", path, cache.size()); - } - - public String get(String key) { return cache.get(key); } - - public void put(String key, String value) { - assert key.indexOf('\t') == -1 : key; - assert key.indexOf('\n') == -1 : key; - assert value.indexOf('\n') == -1 : value; - cache.put(key, value); - if (out != null) { - out.println(key + "\t" + value); - out.flush(); - } - } - - public int size() { return cache.size(); } -} - -/** - * Cache backed by a remote service (see StringCacheServer). - * - * @author Percy Liang - */ -class RemoteStringCache implements StringCache { - private Socket socket; - private PrintWriter out; - private BufferedReader in; - - // Cache things locally. - private FileStringCache local = new FileStringCache(null); - - public RemoteStringCache(String path, String host, int port) { - try { - this.socket = new Socket(host, port); - this.out = new PrintWriter(socket.getOutputStream(), true); - this.in = new BufferedReader(new InputStreamReader(socket.getInputStream())); - makeRequest("open", path, null); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public String makeRequest(String method, String key, String value) { - try { - if (value == null) - out.println(method + "\t" + key); - else - out.println(method + "\t" + key + "\t" + value); - out.flush(); - for(int i = 0; i < 5; ++i) { - try { - String result = in.readLine(); - if (result.equals(StringCacheServer.nullString)) result = null; - return result; - } catch(NullPointerException e) {} - } - throw new NullPointerException(); - } catch (SocketTimeoutException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public String get(String key) { - // First check the local cache. - String value = local.get(key); - if (value == null) - value = makeRequest("get", key, null); - return value; - //return makeRequest("get", key, null); - } - - public void put(String key, String value) { - local.put(key, value); - makeRequest("put", key, value); - } - - public int size() { return local.size(); } -} \ No newline at end of file diff --git a/src/edu/stanford/nlp/sempre/StringValue.java b/src/edu/stanford/nlp/sempre/StringValue.java index 2980034..12de84b 100644 --- a/src/edu/stanford/nlp/sempre/StringValue.java +++ b/src/edu/stanford/nlp/sempre/StringValue.java @@ -19,17 +19,11 @@ public class StringValue extends Value { return tree; } - public double getCompatibility(Value thatValue) { - if (!(thatValue instanceof StringValue)) - return 0; - StringValue that = (StringValue) thatValue; - return this.value.equals(that.value) ? 1 : 0; - } - @Override public int hashCode() { return value.hashCode(); } - @Override public boolean equals(Object thatObj) { - if (!(thatObj instanceof StringValue)) return false; - StringValue that = (StringValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StringValue that = (StringValue) o; return this.value.equals(that.value); } } diff --git a/src/edu/stanford/nlp/sempre/SuperlativeFormula.java b/src/edu/stanford/nlp/sempre/SuperlativeFormula.java index 522afc6..bc23f9d 100644 --- a/src/edu/stanford/nlp/sempre/SuperlativeFormula.java +++ b/src/edu/stanford/nlp/sempre/SuperlativeFormula.java @@ -3,22 +3,24 @@ package edu.stanford.nlp.sempre; import com.google.common.base.Function; import fig.basic.LispTree; +import java.util.List; + /** * Computes the extreme elements of a set |head| according to the degree given * by |relation|. * * @author Percy Liang */ -class SuperlativeFormula extends Formula { - public enum Mode {argmin, argmax}; +public class SuperlativeFormula extends Formula { + public enum Mode { argmin, argmax }; - Mode mode; - int rank; // rank-th item - int count; // Number of items to fetch - Formula head; - Formula relation; // Apply relation(head, degree) and sort by degree. + public final Mode mode; + public final Formula rank; // rank-th item + public final Formula count; // Number of items to fetch + public final Formula head; + public final Formula relation; // Apply relation(head, degree) and sort by degree. - public SuperlativeFormula(Mode mode, int rank, int count, Formula head, Formula relation) { + public SuperlativeFormula(Mode mode, Formula rank, Formula count, Formula head, Formula relation) { this.mode = mode; this.rank = rank; this.count = count; @@ -35,8 +37,8 @@ class SuperlativeFormula extends Formula { public LispTree toLispTree() { LispTree tree = LispTree.proto.newList(); tree.addChild(mode + ""); - tree.addChild(rank + ""); - tree.addChild(count + ""); + tree.addChild(rank.toLispTree()); + tree.addChild(count.toLispTree()); tree.addChild(head.toLispTree()); tree.addChild(relation.toLispTree()); return tree; @@ -44,7 +46,19 @@ class SuperlativeFormula extends Formula { public Formula map(Function func) { Formula result = func.apply(this); - return result == null ? new SuperlativeFormula(mode, rank, count, head.map(func), relation.map(func)) : result; + return result == null ? new SuperlativeFormula(mode, rank.map(func), count.map(func), head.map(func), relation.map(func)) : result; + } + + @Override + public List mapToList(Function> func, boolean alwaysRecurse) { + List res = func.apply(this); + if (res.isEmpty() || alwaysRecurse) { + res.addAll(rank.mapToList(func, alwaysRecurse)); + res.addAll(count.mapToList(func, alwaysRecurse)); + res.addAll(head.mapToList(func, alwaysRecurse)); + res.addAll(relation.mapToList(func, alwaysRecurse)); + } + return res; } @Override @@ -52,18 +66,18 @@ class SuperlativeFormula extends Formula { if (!(thatObj instanceof SuperlativeFormula)) return false; SuperlativeFormula that = (SuperlativeFormula) thatObj; if (this.mode != that.mode) return false; - if (this.rank != that.rank) return false; - if (this.count != that.count) return false; + if (!this.rank.equals(that.rank)) return false; + if (!this.count.equals(that.count)) return false; if (!this.head.equals(that.head)) return false; if (!this.relation.equals(that.relation)) return false; return true; } - + public int computeHashCode() { int hash = 0x7ed55d16; hash = hash * 0xd3a2646c + mode.toString().hashCode(); - hash = hash * 0xd3a2646c + rank; - hash = hash * 0xd3a2646c + count; + hash = hash * 0xd3a2646c + rank.hashCode(); + hash = hash * 0xd3a2646c + count.hashCode(); hash = hash * 0xd3a2646c + head.hashCode(); hash = hash * 0xd3a2646c + relation.hashCode(); return hash; diff --git a/src/edu/stanford/nlp/sempre/TableValue.java b/src/edu/stanford/nlp/sempre/TableValue.java new file mode 100644 index 0000000..d962639 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/TableValue.java @@ -0,0 +1,81 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LispTree; +import fig.basic.LogInfo; +import fig.basic.StrUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents a table (has a header and a list of rows). + * + * (table (State Capital) ((name fb:en.california) (name fb:en.sacramento)) ((name fb:en.oregon) (name fb:en.salem))) + * + * Future: contain information about which columns are important (the head of a + * phrase)? + * + * @author Percy Liang + */ +public class TableValue extends Value { + public final List header; + public final List> rows; + + public int numRows() { return rows.size(); } + public int numCols() { return header.size(); } + + public TableValue(LispTree tree) { + header = new ArrayList(); + rows = new ArrayList>(); + // Read header + LispTree headerTree = tree.child(1); + for (LispTree item : headerTree.children) + header.add(item.value); + // Read rows + for (int i = 2; i < tree.children.size(); i++) { + List row = new ArrayList(); + for (LispTree item : tree.child(i).children) + row.add(Values.fromLispTree(item)); + rows.add(row); + } + } + + public TableValue(List header, List> rows) { + this.header = header; + this.rows = rows; + } + + public LispTree toLispTree() { + LispTree tree = LispTree.proto.newList(); + tree.addChild("table"); + LispTree headerTree = LispTree.proto.newList(); + for (String item : header) + headerTree.addChild(item); + tree.addChild(headerTree); + for (List row : rows) { + LispTree rowTree = LispTree.proto.newList(); + for (Value value : row) + rowTree.addChild(value == null ? LispTree.proto.newLeaf(null) : value.toLispTree()); + tree.addChild(rowTree); + } + return tree; + } + + public void log() { + LogInfo.begin_track("%s", StrUtils.join(header, "\t")); + for (List row : rows) + LogInfo.logs("%s", StrUtils.join(row, "\t")); + LogInfo.end_track(); + } + + // Note: don't compare the headers right now + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableValue that = (TableValue) o; + if (!rows.equals(that.rows)) return false; + return true; + } + + @Override public int hashCode() { return rows.hashCode(); } +} diff --git a/src/edu/stanford/nlp/sempre/TopSemType.java b/src/edu/stanford/nlp/sempre/TopSemType.java new file mode 100644 index 0000000..e92c61c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/TopSemType.java @@ -0,0 +1,12 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LispTree; + +// Represents any possible value. +public class TopSemType extends SemType { + public boolean isValid() { return true; } + public SemType meet(SemType that) { return that; } + public SemType apply(SemType that) { return this; } + public SemType reverse() { return this; } + public LispTree toLispTree() { return LispTree.proto.newLeaf("top"); } +} diff --git a/src/edu/stanford/nlp/sempre/TypeInference.java b/src/edu/stanford/nlp/sempre/TypeInference.java new file mode 100644 index 0000000..32897ec --- /dev/null +++ b/src/edu/stanford/nlp/sempre/TypeInference.java @@ -0,0 +1,251 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.*; +import java.util.*; + +/** + * Performs type inference: given a Formula, return a SemType. + * + * Use a TypeLookup class to look up types of entities and properties. + * - The default NullTypeLookup returns the most generic types. + * - If the Freebase schema is loaded, FreebaseTypeLookup will give + * the type information from the schema. + * - Furthermore, if EntityLexicon is loaded, EntityLexicon will give + * even more refined types. + * + * Note that we just return an upper bound on the type. + * Doesn't have to be perfect, since this is just used to prune out bad combinations. + * + * @author Percy Liang + */ +public final class TypeInference { + private TypeInference() { } + + public static class Options { + @Option(gloss = "Verbosity level") public int verbose = 1; + @Option(gloss = "Class for looking up types") public String typeLookup = "NullTypeLookup"; + } + public static Options opts = new Options(); + + private static TypeLookup typeLookup; + private static TypeLookup getTypeLookup() { + if (typeLookup == null) + typeLookup = (TypeLookup) Utils.newInstanceHard(SempreUtils.resolveClassName(opts.typeLookup)); + return typeLookup; + } + public static void setTypeLookup(TypeLookup typeLookup) { // Kind of hacky, only used in tests + TypeInference.typeLookup = typeLookup; + } + + // For computing type of (call ...) expressions. + private static Map callTypeInfos; + public static void addCallTypeInfo(CallTypeInfo info) { + if (callTypeInfos.containsKey(info.func)) + throw new RuntimeException("Already contains " + info.func); + callTypeInfos.put(info.func, info); + } + private static void initCallTypeInfo() { + if (callTypeInfos != null) return; + callTypeInfos = new HashMap(); + addCallTypeInfo(new CallTypeInfo("Math.cos", ListUtils.newList(SemType.floatType), SemType.floatType)); + addCallTypeInfo(new CallTypeInfo(".concat", ListUtils.newList(SemType.stringType, SemType.stringType), SemType.stringType)); + addCallTypeInfo(new CallTypeInfo(".length", ListUtils.newList(SemType.stringType), SemType.intType)); + addCallTypeInfo(new CallTypeInfo(".toString", ListUtils.newList(SemType.anyType), SemType.stringType)); + // This is just a placeholder now...need to have a more systematic way of + // putting types in (see JavaExecutor). + } + + private static final ValueFormula typeFormula = new ValueFormula(new NameValue(CanonicalNames.TYPE)); + private static class TypeException extends Exception { } + + private static class Env { + private final TypeLookup typeLookup; + private final ImmutableAssocList> list; + private Env(ImmutableAssocList> list, TypeLookup typeLookup) { + this.list = list; + this.typeLookup = typeLookup; + } + public Env(TypeLookup typeLookup) { this(ImmutableAssocList.emptyList, typeLookup); } + + public Env addVar(String var) { + return new Env(list.prepend(var, new Ref(SemType.topType)), typeLookup); + } + public SemType updateType(String var, SemType type) { + Ref ref = list.get(var); + if (ref == null) throw new RuntimeException("Free variable not defined: " + var); + SemType newType = ref.value.meet(type); + if (!newType.isValid() && opts.verbose >= 2) + LogInfo.warnings("Invalid type from [%s MEET %s]", ref.value, type); + ref.value = newType; + return newType; + } + + @Override + public String toString() { + // Used for debugging, so no need to be efficient. + String answer = typeLookup.getClass().getSimpleName() + " {"; + ImmutableAssocList> now = list; + while (!now.isEmpty()) { + answer += now.key + ": " + now.value; + now = now.next; + } + return answer + "}"; + } + } + + // Use the default typeLookup + public static SemType inferType(Formula formula) { + return inferType(formula, getTypeLookup()); + } + + public static SemType inferType(Formula formula, TypeLookup typeLookup) { + SemType type; + try { + type = inferType(formula, new Env(typeLookup), SemType.topType); + } catch (TypeException e) { + type = SemType.bottomType; + } + if (opts.verbose >= 2) LogInfo.logs("TypeInference: %s => %s", formula, type); + return type; + } + + // Unary: fb:domain.type [contains exactly one period] + // Binary: fb:domain.type.property, <, >, etc. + private static boolean isUnary(String s) { + int i = s.indexOf('.'); + if (i == -1) return false; + i = s.indexOf('.', i + 1); + if (i == -1) return true; + return false; + } + + private static SemType check(SemType type) throws TypeException { + if (!type.isValid()) throw new TypeException(); + return type; + } + + // Return the type of |formula| (|type| is an upper bound on the type). + // |env| specifies the mapping form variables to their types. This should be updated. + private static SemType inferType(Formula formula, Env env, SemType type) throws TypeException { + if (opts.verbose >= 5) + LogInfo.logs("TypeInference.inferType(%s, %s, %s)", formula, env, type); + if (formula instanceof VariableFormula) { + return check(env.updateType(((VariableFormula) formula).name, type)); + + } else if (formula instanceof ValueFormula) { + Value value = ((ValueFormula) formula).value; + if (value instanceof NumberValue) return check(type.meet(SemType.numberType)); + else if (value instanceof StringValue) return check(type.meet(SemType.stringType)); + else if (value instanceof DateValue) return check(type.meet(SemType.dateType)); + else if (value instanceof NameValue) { + String id = ((NameValue) value).id; + + if (isUnary(id)) { // Unary + SemType unaryType = env.typeLookup.getEntityType(id); + if (unaryType == null) + unaryType = SemType.entityType; + type = check(type.meet(unaryType)); + } else { // Binary + // Careful of the reversal. + SemType propertyType = null; + if (!CanonicalNames.isReverseProperty(id)) { + propertyType = env.typeLookup.getPropertyType(id); + } else { + propertyType = env.typeLookup.getPropertyType(CanonicalNames.reverseProperty(id)); + if (propertyType != null) propertyType = propertyType.reverse(); + } + if (propertyType == null) + propertyType = SemType.anyAnyFunc; // Don't know + type = check(type.meet(propertyType)); + } + return type; + } else { + throw new RuntimeException("Unhandled value: " + value); + } + + } else if (formula instanceof JoinFormula) { + JoinFormula join = (JoinFormula) formula; + + // Special case: (fb:type.object.type fb:people.person) => fb:people.person + if (typeFormula.equals(join.relation) && join.child instanceof ValueFormula) + return check(type.meet(SemType.newAtomicSemType(Formulas.getString(join.child)))); + + SemType relationType = inferType(join.relation, env, new FuncSemType(SemType.topType, type)); // Relation + SemType childType = inferType(join.child, env, relationType.getArgType()); // Child + relationType = inferType(join.relation, env, new FuncSemType(childType, type)); // Relation again + return check(relationType.getRetType()); + + } else if (formula instanceof MergeFormula) { + MergeFormula merge = (MergeFormula) formula; + type = check(type.meet(SemType.anyType)); // Must be not higher-order + type = inferType(merge.child1, env, type); + type = inferType(merge.child2, env, type); + return type; + + } else if (formula instanceof MarkFormula) { + MarkFormula mark = (MarkFormula) formula; + env = env.addVar(mark.var); + type = check(type.meet(SemType.anyType)); // Must be not higher-order + type = inferType(mark.body, env, type); + type = check(env.updateType(mark.var, type)); + return type; + + } else if (formula instanceof LambdaFormula) { + LambdaFormula lambda = (LambdaFormula) formula; + env = env.addVar(lambda.var); + SemType bodyType = inferType(lambda.body, env, type.getRetType()); + SemType varType = check(env.updateType(lambda.var, type.getArgType())); + return new FuncSemType(varType, bodyType); + + } else if (formula instanceof NotFormula) { + NotFormula not = (NotFormula) formula; + type = check(type.meet(SemType.anyType)); // Must be not higher-order + return inferType(not.child, env, type); + + } else if (formula instanceof AggregateFormula) { + AggregateFormula aggregate = (AggregateFormula) formula; + SemType childType = inferType(aggregate.child, env, SemType.anyType); + return check(SemType.numberType.meet(type)); + + } else if (formula instanceof ArithmeticFormula) { + ArithmeticFormula arith = (ArithmeticFormula) formula; + // TODO(pliang): allow date + duration + type = inferType(arith.child1, env, type); + type = inferType(arith.child2, env, type); + return check(type.meet(SemType.numberOrDateType)); + + } else if (formula instanceof ReverseFormula) { + ReverseFormula reverse = (ReverseFormula) formula; + SemType reverseType = inferType(reverse.child, env, type.reverse()); + return check(reverseType.reverse()); + + } else if (formula instanceof SuperlativeFormula) { + SuperlativeFormula superlative = (SuperlativeFormula) formula; + inferType(superlative.rank, env, SemType.numberType); + inferType(superlative.count, env, SemType.numberType); + type = check(type.meet(SemType.anyType)); // Must be not higher-order + type = inferType(superlative.head, env, type); // Head + SemType relationType = inferType(superlative.relation, env, new FuncSemType(SemType.numberOrDateType, type)); // Relation + type = inferType(superlative.head, env, relationType.getRetType()); // Head again + return type; + + } else if (formula instanceof CallFormula) { + initCallTypeInfo(); + CallFormula call = (CallFormula) formula; + if (!(call.func instanceof ValueFormula)) return SemType.bottomType; + Value value = ((ValueFormula) call.func).value; + if (!(value instanceof NameValue)) return SemType.bottomType; + String func = ((NameValue) value).id; + + CallTypeInfo info = callTypeInfos.get(func); + if (info == null) return SemType.anyType; // Don't know + + if (info.argTypes.size() != call.args.size()) return SemType.bottomType; + for (int i = 0; i < info.argTypes.size(); i++) + inferType(call.args.get(i), env, info.argTypes.get(i)); + return check(type.meet(info.retType)); + } else { + throw new RuntimeException("Can't infer type of formula: " + formula); + } + } +} diff --git a/src/edu/stanford/nlp/sempre/TypeLookup.java b/src/edu/stanford/nlp/sempre/TypeLookup.java new file mode 100644 index 0000000..d31fd4b --- /dev/null +++ b/src/edu/stanford/nlp/sempre/TypeLookup.java @@ -0,0 +1,17 @@ +package edu.stanford.nlp.sempre; + +/** + * A TypeLookup object handles the domain-specific part of type inference. + * TypeInference handles the domain general part. + * + * Given an entity or a property, return the appropriate SemType. + */ +public interface TypeLookup { + // e.g., fb:en.barack_obama => (union fb:people.person ...) + // Return null if unknown + SemType getEntityType(String entity); + + // e.g., fb:people.person.place_of_birth => (-> fb:location.location fb:people.person) + // Return null if unknown + SemType getPropertyType(String property); +} diff --git a/src/edu/stanford/nlp/sempre/UnionSemType.java b/src/edu/stanford/nlp/sempre/UnionSemType.java new file mode 100644 index 0000000..cc1dcd2 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/UnionSemType.java @@ -0,0 +1,64 @@ +package edu.stanford.nlp.sempre; + +import fig.basic.LispTree; +import java.util.*; + +// Represents the union of a set of base types. +public class UnionSemType extends SemType { + public final List baseTypes; + public boolean isValid() { return baseTypes.size() > 0; } + + // Constructors + public UnionSemType() { + this.baseTypes = new ArrayList(); + } + public UnionSemType(SemType... baseTypes) { + this.baseTypes = new ArrayList(); + for (SemType baseType : baseTypes) + if (baseType.isValid()) + this.baseTypes.add(baseType); + } + public UnionSemType(Collection baseTypes) { + this.baseTypes = new ArrayList(); + for (SemType baseType : baseTypes) + if (baseType.isValid()) + this.baseTypes.add(baseType); + } + + public SemType meet(SemType that) { + if (that instanceof TopSemType) return this; + List result = new ArrayList<>(); + for (SemType baseType : baseTypes) + result.add(baseType.meet(that)); + return new UnionSemType(result).simplify(); + } + + public SemType apply(SemType that) { + List result = new ArrayList<>(); + for (SemType baseType : baseTypes) + result.add(baseType.apply(that)); + return new UnionSemType(result).simplify(); + } + + public SemType reverse() { + List result = new ArrayList<>(); + for (SemType baseType : baseTypes) + result.add(baseType.reverse()); + return new UnionSemType(result).simplify(); + } + + public LispTree toLispTree() { + LispTree result = LispTree.proto.newList(); + result.addChild("union"); + for (SemType baseType : baseTypes) + result.addChild(baseType.toLispTree()); + return result; + } + + public SemType simplify() { + if (baseTypes.size() == 0) return SemType.bottomType; + if (baseTypes.size() == 1) return baseTypes.get(0); + if (baseTypes.contains(SemType.topType)) return SemType.topType; + return this; + } +} diff --git a/src/edu/stanford/nlp/sempre/UriValue.java b/src/edu/stanford/nlp/sempre/UriValue.java index 22465e3..baec2d6 100644 --- a/src/edu/stanford/nlp/sempre/UriValue.java +++ b/src/edu/stanford/nlp/sempre/UriValue.java @@ -19,16 +19,12 @@ public class UriValue extends Value { tree.addChild(value != null ? value : ""); return tree; } - public double getCompatibility(Value thatValue) { - if (!(thatValue instanceof UriValue)) - return 0.0; - UriValue that = (UriValue) thatValue; - return this.value.equals(that.value) ? 1.0 : 0.0; - } @Override public int hashCode() { return value.hashCode(); } - @Override public boolean equals(Object thatObj) { - UriValue that = (UriValue)thatObj; + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UriValue that = (UriValue) o; return this.value.equals(that.value); } } diff --git a/src/edu/stanford/nlp/sempre/Value.java b/src/edu/stanford/nlp/sempre/Value.java index 0b50fab..1662bcb 100644 --- a/src/edu/stanford/nlp/sempre/Value.java +++ b/src/edu/stanford/nlp/sempre/Value.java @@ -13,14 +13,6 @@ import fig.basic.LogInfo; public abstract class Value { public abstract LispTree toLispTree(); - // |this| is target value, |that| is predicted value - // Return a number [0,1] denoting how good |that| is correct. - // Default implementation: just test for equality. - // Subclasses should override with other. - public double getCompatibility(Value that) { - return this.equals(that) ? 1 : 0; - } - // Print using LogInfo. public void log() { LogInfo.logs("%s", toString()); } @@ -32,6 +24,6 @@ public abstract class Value { return Values.fromLispTree(LispTree.proto.parseFromString(str)); } - @Override abstract public boolean equals(Object o); - @Override abstract public int hashCode(); + @Override public abstract boolean equals(Object o); + @Override public abstract int hashCode(); } diff --git a/src/edu/stanford/nlp/sempre/ValueEvaluator.java b/src/edu/stanford/nlp/sempre/ValueEvaluator.java new file mode 100644 index 0000000..7bd790a --- /dev/null +++ b/src/edu/stanford/nlp/sempre/ValueEvaluator.java @@ -0,0 +1,10 @@ +package edu.stanford.nlp.sempre; + +/** + * Given a target denotation Value and a predicted denotation Value, + * return a compatibility. + */ +public interface ValueEvaluator { + // Return a number [0, 1] that denotes how well we're doing. + double getCompatibility(Value target, Value pred); +} diff --git a/src/edu/stanford/nlp/sempre/ValueFormula.java b/src/edu/stanford/nlp/sempre/ValueFormula.java index fe1ab19..0ab2847 100644 --- a/src/edu/stanford/nlp/sempre/ValueFormula.java +++ b/src/edu/stanford/nlp/sempre/ValueFormula.java @@ -13,7 +13,7 @@ public class ValueFormula extends PrimitiveFormula { public ValueFormula(T value) { this.value = value; } public LispTree toLispTree() { - if (value instanceof NameValue) return LispTree.proto.newLeaf(((NameValue)value).id); + if (value instanceof NameValue) return LispTree.proto.newLeaf(((NameValue) value).id); return value.toLispTree(); } diff --git a/src/edu/stanford/nlp/sempre/Values.java b/src/edu/stanford/nlp/sempre/Values.java index 62a5972..9a4692d 100644 --- a/src/edu/stanford/nlp/sempre/Values.java +++ b/src/edu/stanford/nlp/sempre/Values.java @@ -7,20 +7,36 @@ import fig.basic.LispTree; * * @author Percy Liang */ -public class Values { - public static Value fromLispTree(LispTree tree) { +public final class Values { + private Values() { } + + // Try to parse the LispTree into a value. + // If it fails, just return null. + public static Value fromLispTreeOrNull(LispTree tree) { if (tree.isLeaf()) - throw new RuntimeException("Invalid value: " + tree); + return null; String type = tree.child(0).value; - if (type.equals("name")) return new NameValue(tree); - if (type.equals("boolean")) return new BooleanValue(tree); - if (type.equals("number")) return new NumberValue(tree); - if (type.equals("string")) return new StringValue(tree); - if (type.equals("list")) return new ListValue(tree); - if (type.equals("description")) return new DescriptionValue(tree); - if (type.equals("url")) return new UriValue(tree); - if (type.equals("date")) return new DateValue(tree); - if (type.equals("error")) return new ErrorValue(tree); - throw new RuntimeException("Invalid value: " + tree); + if ("name".equals(type)) return new NameValue(tree); + if ("boolean".equals(type)) return new BooleanValue(tree); + if ("number".equals(type)) return new NumberValue(tree); + if ("string".equals(type)) return new StringValue(tree); + if ("list".equals(type)) return new ListValue(tree); + if ("table".equals(type)) return new TableValue(tree); + if ("description".equals(type)) return new DescriptionValue(tree); + if ("url".equals(type)) return new UriValue(tree); + if ("context".equals(type)) return new ContextValue(tree); + if ("date".equals(type)) return new DateValue(tree); + if ("error".equals(type)) return new ErrorValue(tree); + return null; } + + // Try to parse. If it fails, throw an exception. + public static Value fromLispTree(LispTree tree) { + Value value = fromLispTreeOrNull(tree); + if (value == null) + throw new RuntimeException("Invalid value: " + tree); + return value; + } + + public static Value fromString(String s) { return fromLispTree(LispTree.proto.parseFromString(s)); } } diff --git a/src/edu/stanford/nlp/sempre/VariableFormula.java b/src/edu/stanford/nlp/sempre/VariableFormula.java index dd02aee..7e6f679 100644 --- a/src/edu/stanford/nlp/sempre/VariableFormula.java +++ b/src/edu/stanford/nlp/sempre/VariableFormula.java @@ -7,7 +7,7 @@ import fig.basic.LispTree; * * @author Percy Liang */ -class VariableFormula extends PrimitiveFormula { +public class VariableFormula extends PrimitiveFormula { public final String name; // Name of variable. public VariableFormula(String name) { this.name = name; } public LispTree toLispTree() { return LispTree.proto.newList("var", name); } @@ -18,6 +18,6 @@ class VariableFormula extends PrimitiveFormula { VariableFormula that = (VariableFormula) thatObj; return this.name.equals(that.name); } - + public int computeHashCode() { return name.hashCode(); } } diff --git a/src/edu/stanford/nlp/sempre/Vis.java b/src/edu/stanford/nlp/sempre/Vis.java deleted file mode 100644 index 9a796c5..0000000 --- a/src/edu/stanford/nlp/sempre/Vis.java +++ /dev/null @@ -1,233 +0,0 @@ -package edu.stanford.nlp.sempre; - -import com.google.common.base.Joiner; -import edu.stanford.nlp.sempre.vis.BeamFigures; -import edu.stanford.nlp.sempre.vis.ConfusionMatrices; -import edu.stanford.nlp.sempre.vis.ExampleDerivations; -import edu.stanford.nlp.sempre.vis.Utils; -import fig.basic.IOUtils; -import fig.basic.LispTree; -import fig.basic.LogInfo; -import fig.basic.Option; -import fig.exec.Execution; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileFilter; -import java.io.PrintWriter; -import java.util.*; - -/** - * Entry point for debugging and vis tools for the semantic parser. - * - * @author Roy Frostig - */ -public class Vis implements Runnable { - public static final class Options { - @Option public List execPaths = null; - @Option public int topN = 10; - @Option public int iter = -1; - @Option public String group = null; - - @Option public String command = ""; - - @Option public String in = null; - @Option public int width = 100; - - @Option public boolean useJson = true; - } - - public static final Options opts = new Options(); - - public static void main(String[] args) { - Execution.run(args, "Main", new Vis(), Master.getOptionsParser()); - } - - // -- Commands -- - - private void requireExecPaths() { - if (opts.execPaths == null) - throw new RuntimeException("Need -execPaths"); - if (opts.execPaths.isEmpty()) - return; - } - - private void cmp() { - requireExecPaths(); - if (opts.iter >= 0 && opts.group != null) - new ExampleDerivations(opts.execPaths, opts.topN).write(opts.iter, opts.group); - else - new ExampleDerivations(opts.execPaths, opts.topN).writeAll(); - } - - private void confuse() { - requireExecPaths(); - new ConfusionMatrices(opts.execPaths).logsAll(); - } - - private void beamfigs() { - requireExecPaths(); - new BeamFigures(opts.execPaths).writeAll(); - } - - private void lispTreePrettyPrint() { - String s = ""; - if (opts.in == null) { - Scanner stdin = new Scanner(System.in); - while (stdin.hasNextLine()) - s += stdin.nextLine(); - stdin.close(); - } else { - s = Joiner.on("").join(IOUtils.readLinesHard(opts.in)); - } - LogInfo.log(LispTree.proto.parseFromString(s).toStringWrap(opts.width)); - } - - public void run() { - Execution.putOutput("vis", true); - - if ("cmp".equals(opts.command)) { - cmp(); - } else if ("confuse".equals(opts.command)) { - confuse(); - } else if ("beamfigs".equals(opts.command)) { - beamfigs(); - } else if ("ltpp".equals(opts.command)) { - lispTreePrettyPrint(); - } - } - - @SuppressWarnings({"deprecation"}) - public static void writeExamples(int iter, String group, - List examples, - boolean outputPredDerivations) { - String basePath = "preds-iter" + iter + "-" + group + ".examples"; - String outPath = Execution.getFile(basePath); - if (outPath == null || examples.size() == 0) return; - LogInfo.begin_track("Writing examples to %s", basePath); - PrintWriter out = IOUtils.openOutHard(outPath); - if (opts.useJson) { - Json.writeValueHard( - out, - Collections.singletonMap("examples", examples), - outputPredDerivations ? Example.JsonViews.WithDerivations.class : Object.class); - } else { - // Deprecation warnings come through here. - for (Example ex : examples) { - if (outputPredDerivations) - out.println(ex.toLispTree(true).toString()); - else - out.println(ex.toLispTree(false).toStringWrap(0, 1000)); - } - } - out.close(); - LogInfo.end_track(); - } - - public static List getFilesPerExec(List execPaths, final int iter, final String group) { - List files = new ArrayList(); - for (String execPath : execPaths) { - List lsFiles = IOUtils.getFilesUnder( - execPath, new FileFilter() { - public boolean accept(File f) { - if (f.isDirectory()) - return true; - return f.getName().startsWith("preds-iter" + iter + "-" + group); - } - }); - for (int i = 0; i < lsFiles.size(); i++) { - if (lsFiles.get(i).isDirectory()) { - lsFiles.remove(i); - i--; - } - } - if (lsFiles.isEmpty()) - return null; - sortFiles(lsFiles); - files.add(lsFiles.get(0)); - } - return files; - } - - public static File getExecFile(String execPath, final int iter, final String group) { - List files = getFilesPerExec(Collections.singletonList(execPath), iter, group); - if (files == null) - return null; - return files.get(0); - } - - public static List getExecIterFiles(String execPath, final String group) { - List files = new ArrayList(); - for (int iter = 0; ; iter++) { - File file = Vis.getExecFile(execPath, iter, group); - if (file == null) - break; - files.add(file); - } - return files; - } - - /** - * @param files List of ".examples" files. - * @return Iterable over rows of examples, each row taking the subsequent - * example from each file. - */ - @SuppressWarnings({"deprecation"}) - public static Iterable> zipExamples(final List files) { - // Wish list: lazy IO in this language. - return new edu.stanford.nlp.sempre.vis.Utils.SimpleGenerator>() { - List readers = null; - - private void closeFiles() { - if (readers != null) - for (BufferedReader in : readers) - edu.stanford.nlp.sempre.vis.Utils.closeHard(in); - } - - private void openFiles() { - readers = new ArrayList(); - for (File file : files) - readers.add(IOUtils.openInHard(file)); - } - - @Override - protected List computeNext() { - if (readers == null) - openFiles(); - List row = new ArrayList(readers.size()); - for (BufferedReader in : readers) { - String line = edu.stanford.nlp.sempre.vis.Utils.readLineHard(in); - if (line == null) { - closeFiles(); - return null; - } - LispTree tree = LispTree.proto.parseFromString(line); - // Deprecation warnings come through here. - Example ex = Example.fromLispTree(tree); - row.add(ex); - } - return row; - } - }; - } - - public static Iterable getExamples(final File file) { - final Iterator> examples = zipExamples(Collections.singletonList(file)).iterator(); - return new Utils.SimpleGenerator() { - @Override - protected Example computeNext() { - return examples.hasNext() ? examples.next().get(0) : null; - } - }; - } - - private static void sortFiles(List files) { - Collections.sort( - files, new Comparator() { - @Override - public int compare(File a, File b) { - return a.getName().compareTo(b.getName()); - } - }); - } -} diff --git a/src/edu/stanford/nlp/sempre/build.xml b/src/edu/stanford/nlp/sempre/build.xml new file mode 100644 index 0000000..20bc99a --- /dev/null +++ b/src/edu/stanford/nlp/sempre/build.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/src/edu/stanford/nlp/sempre/cache/FileStringCache.java b/src/edu/stanford/nlp/sempre/cache/FileStringCache.java new file mode 100644 index 0000000..adfcffe --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/FileStringCache.java @@ -0,0 +1,168 @@ +package edu.stanford.nlp.sempre.cache; + +import fig.basic.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Cache backed by a file. + * + * @author Percy Liang + */ +public class FileStringCache implements StringCache, LruCallback { + public static class Options { + @Option(gloss = "Cache capacity (in MB)") + public int capacity = 35 * 1024; + + @Option(gloss = "Auto-flush cache to disk every N accesses") + public int flushFrequency = Integer.MAX_VALUE; + + @Option(gloss = "Append mode instead of dump mode") + public boolean appendMode = true; + + public int verbose = 0; + } + public static final Options opts = new Options(); + + private String path; + private PrintWriter out; + + private final LinkedHashMap cache; + private final StatFig keyStats = new StatFig(); + private final StatFig valStats = new StatFig(); + private int numTouches = 0; + private int numEvictions = 0; + + public FileStringCache() { + int cap = opts.capacity; + cap = (cap < 0) ? cap : (cap * 1024 * 1024); + if (cap < 0) { + cache = new LinkedHashMap(); + } else { + cache = new LruMap(cap, this); + } + } + + public String getPath() { return path; } + + public void init(String path) { + if (this.path != null) throw new RuntimeException("Already initialized with " + this.path); + this.path = path; + + // Read existing. + if (new File(path).exists()) { + try { + BufferedReader in = IOUtils.openInHard(path); + String line; + while ((line = in.readLine()) != null) { + String[] tokens = line.split("\t", 2); + if (tokens.length != 2) + throw new RuntimeException("Invalid line in cache file: " + line); + cache.put(tokens[0], tokens[1]); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + LogInfo.logs("Using cache %s (%d entries)", path, cache.size()); + + if (opts.appendMode) + out = IOUtils.openOutAppendHard(path); + flush(); + } + + private void flush() { + if (out != null) // Append mode + return; + + if (path == null) // No file-backing + return; + + if (opts.verbose >= 2) { + LogInfo.begin_track("FileStringCache FLUSH (dump mode)"); + LogInfo.logs("Size: %d", size()); + if (cache instanceof LruMap) + LogInfo.logs("Memory: %d", ((LruMap) cache).getBytes()); + LogInfo.logs("Touches: %d", numTouches); + LogInfo.logs("Evictions: %d", numEvictions); + LogInfo.logs("Evicted keys: %s", keyStats); + LogInfo.logs("Evicted values: %s", valStats); + LogInfo.end_track(); + } + + PrintWriter dumpOut = IOUtils.openOutHard(this.path + ".tmp"); + for (Map.Entry entry : cache.entrySet()) { + dumpOut.println(entry.getKey() + "\t" + entry.getValue()); + } + dumpOut.flush(); + dumpOut.close(); + try { + Path src = FileSystems.getDefault().getPath(this.path + ".tmp"); + Path dst = FileSystems.getDefault().getPath(this.path); + Files.move(src, dst, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String get(String key) { return cache.get(key); } + + public void put(String key, String value) { + assert key.indexOf('\t') == -1 : key; + assert key.indexOf('\n') == -1 : key; + assert value.indexOf('\n') == -1 : value; + if (opts.verbose >= 5) { + logTrack("FileStringCache PUT (before)", key, value); + } + cache.put(key, value); + if (out != null) { // Append mode + out.println(key + "\t" + value); + out.flush(); + } + if (numTouches++ % opts.flushFrequency == 0) + flush(); + } + + public int size() { return cache.size(); } + + @Override + public void onEvict(Map.Entry entry) { + if (opts.verbose >= 5) { + logTrack("FileStringCache EVICT (after)", entry.getKey(), entry.getValue()); + } + numEvictions++; + keyStats.add(entry.getKey(), entry.getKey().length()); + valStats.add(entry.getValue(), entry.getValue().length()); + } + + private void logTrack(String header, String key, String value) { + LogInfo.begin_track(header); + LogInfo.logs("Key size: %d (%d bytes)", key.length(), MemUsage.getBytes(key)); + LogInfo.logs("Val size: %d (%d bytes)", value.length(), MemUsage.getBytes(value)); + if (cache instanceof LruMap) { + LogInfo.logs("Cache size: %d entries (%d bytes of %d)", + cache.size(), + ((LruMap) cache).getBytes(), + ((LruMap) cache).getCapacity()); + } + LogInfo.end_track(); + } + + // For tests + @Deprecated + public int getNumTouches() { + return numTouches; + } +} diff --git a/src/edu/stanford/nlp/sempre/cache/LruCallback.java b/src/edu/stanford/nlp/sempre/cache/LruCallback.java new file mode 100644 index 0000000..ce92ca5 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/LruCallback.java @@ -0,0 +1,10 @@ +package edu.stanford.nlp.sempre.cache; + +import java.util.Map; + +/** + * @author Roy Frostig + */ +public interface LruCallback { + void onEvict(Map.Entry entry); +} diff --git a/src/edu/stanford/nlp/sempre/cache/LruMap.java b/src/edu/stanford/nlp/sempre/cache/LruMap.java new file mode 100644 index 0000000..6421f0c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/LruMap.java @@ -0,0 +1,86 @@ +package edu.stanford.nlp.sempre.cache; + +import fig.basic.MemUsage; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * TODO(rf): Fig.MemUsage is buggy wrt computing LinkedHashMap byte + * usage, so we do our own bookkeeping here, but we are doing so very + * incompletely (i.e. only on put() and remove()). + * + * @author Roy Frostig + */ +public class LruMap extends LinkedHashMap { + private final int cap; + private final LruCallback callback; + private int bytes = 0; + + public LruMap(int capacity) { + this(capacity, null); + } + + public LruMap(int capacity, LruCallback evictCallback) { + // "As a general rule, the default load factor (.75) offers a good + // tradeoff between time and space costs." + // -- Java 8 API, + // http://docs.oracle.com/javase/8/docs/api/java/util/HashMap.html + super(capacity, 0.75f, true); // Flag true for access-order. + this.cap = capacity; + this.callback = evictCallback; + } + + public int getCapacity() { + return cap; + } + + public int getBytes() { + return bytes; + } + + @Override + public V put(K key, V value) { + boolean replacing = containsKey(key); + V old = super.get(key); + bytes += MemUsage.getBytes(value); + if (replacing) { + bytes -= MemUsage.getBytes(old); + } else { + bytes += MemUsage.getBytes(key); + } + return super.put(key, value); + } + + @Override + public V remove(Object key) { + boolean decr = containsKey(key); + V old = super.remove(key); + if (decr) { + bytes -= MemUsage.getBytes(key); + bytes -= MemUsage.getBytes(old); + } + return old; + } + + /** + * Ignore the argument, iterate in access order and remove until + * memory constraints are satisfied. Always return false, since + * we did our own removal. + */ + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + while (getBytes() > cap && !isEmpty()) { + Map.Entry toRemove = null; + for (Map.Entry entry : entrySet()) { + toRemove = entry; + break; + } + remove(toRemove.getKey()); + if (callback != null) + callback.onEvict(toRemove); + } + return false; + } +} + diff --git a/src/edu/stanford/nlp/sempre/cache/RemoteStringCache.java b/src/edu/stanford/nlp/sempre/cache/RemoteStringCache.java new file mode 100644 index 0000000..43278ec --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/RemoteStringCache.java @@ -0,0 +1,82 @@ +package edu.stanford.nlp.sempre.cache; + +import java.io.*; +import java.net.*; + +import fig.basic.LogInfo; + +/** + * Cache backed by a remote service (see StringCacheServer). + * + * @author Percy Liang + */ +public class RemoteStringCache implements StringCache { + public static final int NUM_TRIES = 5; + + private Socket socket; + private PrintWriter out; + private BufferedReader in; + + // Cache things locally. + private FileStringCache local = new FileStringCache(); + + public RemoteStringCache(String path, String host, int port) { + try { + LogInfo.begin_track("RemoteStringCache: connecting to %s:%s to access %s", host, port, path); + this.socket = new Socket(host, port); + this.out = new PrintWriter(socket.getOutputStream(), true); + this.in = new BufferedReader(new InputStreamReader(socket.getInputStream())); + String response = makeRequest("open", path, null); + LogInfo.logs("Using cache path=%s, host=%s, port=%s", path, host, port); + if (!response.equals("OK")) { + throw new RuntimeException(response); + } + LogInfo.end_track(); + } catch (UnknownHostException e) { + LogInfo.end_track(); + throw new RuntimeException(e); + } catch (IOException e) { + LogInfo.end_track(); + throw new RuntimeException(e); + } + } + + public String makeRequest(String method, String key, String value) { + try { + if (value == null) + out.println(method + "\t" + key); + else + out.println(method + "\t" + key + "\t" + value); + out.flush(); + for (int i = 0; i < NUM_TRIES; i++) { + try { + String result = in.readLine(); + if (result.equals(StringCacheServer.nullString)) result = null; + return result; + } catch (NullPointerException e) { + LogInfo.logs("RemoteStringCache.makeRequest(%s, %s, %s) failed", method, key, value); + } + } + throw new NullPointerException(); + } catch (SocketTimeoutException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String get(String key) { + // First check the local cache. + String value = local.get(key); + if (value == null) + value = makeRequest("get", key, null); + return value; + } + + public void put(String key, String value) { + local.put(key, value); + makeRequest("put", key, value); + } + + public int size() { return local.size(); } +} diff --git a/src/edu/stanford/nlp/sempre/cache/StringCache.java b/src/edu/stanford/nlp/sempre/cache/StringCache.java new file mode 100644 index 0000000..4ce1f9e --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/StringCache.java @@ -0,0 +1,12 @@ +package edu.stanford.nlp.sempre.cache; + +/** + * Stores a Map, which is synchronized with disk. + * + * @author Percy Liang + */ +public interface StringCache { + String get(String key); + void put(String key, String value); +} + diff --git a/src/edu/stanford/nlp/sempre/StringCacheServer.java b/src/edu/stanford/nlp/sempre/cache/StringCacheServer.java similarity index 83% rename from src/edu/stanford/nlp/sempre/StringCacheServer.java rename to src/edu/stanford/nlp/sempre/cache/StringCacheServer.java index 776ec86..92c1ae4 100644 --- a/src/edu/stanford/nlp/sempre/StringCacheServer.java +++ b/src/edu/stanford/nlp/sempre/cache/StringCacheServer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.cache; import fig.basic.LogInfo; import fig.basic.Option; @@ -31,7 +31,7 @@ public class StringCacheServer implements Runnable { private HashMap caches = new HashMap(); private boolean terminated = false; - // Represents the null value. + // Represents the null value to be returned back to the user. public static String nullString = "__NULL__"; class ClientHandler implements Runnable { @@ -49,19 +49,32 @@ public class StringCacheServer implements Runnable { String line; int numGets = 0, numPuts = 0, numErrors = 0; while (!terminated && (line = in.readLine()) != null) { - //LogInfo.logs("Input: %s", line); + // LogInfo.logs("Input: %s", line); String[] tokens = line.split("\t"); String response = null; if (tokens[0].equals("open") && tokens.length == 2) { String path = tokens[1]; + // Create the cache if necessary synchronized (caches) { cache = caches.get(path); if (cache == null) { - cache = new FileStringCache(path); + cache = new FileStringCache(); caches.put(path, cache); } } response = "OK"; + synchronized (cache) { + if (cache.getPath() == null) { + LogInfo.begin_track("Loading %s", path); + try { + cache.init(path); + } catch (Throwable t) { + response = "ERROR: " + t; + } + LogInfo.logs("Response: %s", response); + LogInfo.end_track(); + } + } } else if (tokens[0].equals("get") && tokens.length == 2) { if (cache == null) { response = "ERROR: no file opened yet"; @@ -93,7 +106,7 @@ public class StringCacheServer implements Runnable { response = "ERROR: " + line; numErrors++; } - //LogInfo.logs("Response: %s", response); + // LogInfo.logs("Response: %s", response); out.println(response); out.flush(); } @@ -123,6 +136,8 @@ public class StringCacheServer implements Runnable { } public static void main(String[] args) throws Exception { - Execution.run(args, new StringCacheServer()); + Execution.run(args, + new StringCacheServer(), + "FileStringCache", FileStringCache.opts); } } diff --git a/src/edu/stanford/nlp/sempre/StringCacheUtils.java b/src/edu/stanford/nlp/sempre/cache/StringCacheUtils.java similarity index 64% rename from src/edu/stanford/nlp/sempre/StringCacheUtils.java rename to src/edu/stanford/nlp/sempre/cache/StringCacheUtils.java index 13d6803..51c7700 100644 --- a/src/edu/stanford/nlp/sempre/StringCacheUtils.java +++ b/src/edu/stanford/nlp/sempre/cache/StringCacheUtils.java @@ -1,10 +1,13 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.cache; -public class StringCacheUtils { + +public final class StringCacheUtils { + private StringCacheUtils() { } // description could be - // Local path: + // Local path: ... // Remote path: jacko:4000:/u/nlp/... public static StringCache create(String description) { + // Remote if (description != null && description.indexOf(':') != -1) { String[] tokens = description.split(":", 3); if (tokens.length != 3) @@ -14,6 +17,9 @@ public class StringCacheUtils { } // Local - return new FileStringCache(description); + FileStringCache cache = new FileStringCache(); + if (description != null) + cache.init(description); + return cache; } -} \ No newline at end of file +} diff --git a/src/edu/stanford/nlp/sempre/cache/build.xml b/src/edu/stanford/nlp/sempre/cache/build.xml new file mode 100644 index 0000000..f1e217a --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/build.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/edu/stanford/nlp/sempre/cache/test/StringCacheTest.java b/src/edu/stanford/nlp/sempre/cache/test/StringCacheTest.java new file mode 100644 index 0000000..a001339 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/cache/test/StringCacheTest.java @@ -0,0 +1,112 @@ +package edu.stanford.nlp.sempre.cache.test; + +import edu.stanford.nlp.sempre.cache.FileStringCache; +import fig.basic.IOUtils; +import fig.basic.MemUsage; + +import org.testng.annotations.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.testng.AssertJUnit.assertEquals; + +/** + * @author Roy Frostig + */ +public class StringCacheTest { + private int numLines(BufferedReader in) throws IOException { + int n = 0; + while (in.readLine() != null) n++; + return n; + } + + /** + * Check that we flush at the given frequency + * @throws IOException + */ + @SuppressWarnings("deprecation") + @Test(groups = "fs") + public void testFlush() throws IOException { + FileStringCache.opts.appendMode = false; + FileStringCache.opts.capacity = 1; + FileStringCache.opts.flushFrequency = 10; + FileStringCache.opts.verbose = 5; + + final String fs = "StringCacheTest-cache.tmp"; + final Path fsPath = FileSystems.getDefault().getPath(fs); + + Files.deleteIfExists(fsPath); + FileStringCache cache = new FileStringCache(); + cache.init(fs); + + for (int i = 0; i <= 100; i++) { + String key = "key:" + i; + String val = "val:" + i; + cache.put(key, val); + assertEquals(cache.getNumTouches(), i + 1); + if (i > 0 && i % 10 == 0) { + int lines = numLines(IOUtils.openInHard(fs)); + System.out.println("!!! " + lines + " = " + i); + assertEquals(lines, i + 1); + } + } + Files.deleteIfExists(fsPath); + } + + /** + * Check that we really do evict at the capacity. + * @throws IOException + */ + @Test(groups = "fs") + public void testEvict() throws IOException { + FileStringCache.opts.appendMode = false; + FileStringCache.opts.capacity = 10; + FileStringCache.opts.flushFrequency = 10; + FileStringCache.opts.verbose = 5; + + final String fs = "StringCacheTest-cache.tmp"; + final Path fsPath = FileSystems.getDefault().getPath(fs); + + Files.deleteIfExists(fsPath); + FileStringCache cache = new FileStringCache(); + cache.init(fs); + + // Make ~20 MB of string data + String junk20MB = "junk"; + while (MemUsage.getBytes(junk20MB) <= 20 * 1024 * 1024) + junk20MB += junk20MB; + + // Make ~1 MB of string data + String junk1MB = "junk"; + while (MemUsage.getBytes(junk1MB) <= 1024 * 1024) + junk1MB += junk1MB; + + // Add something small, to be evicted + cache.put(junk1MB + "1", "test1"); + assertEquals(1, cache.size()); + assert cache.get(junk1MB + "1").equals("test1"); + + // Add something small, shouldn't evict anything + cache.put(junk1MB + "2", "test2"); + assertEquals(2, cache.size()); + assert cache.get(junk1MB + "1").equals("test1"); + assert cache.get(junk1MB + "2").equals("test2"); + + // Add something big, to saturate cache and kick out the small entries. + // Also kicks itself out. + cache.put(junk20MB + "1", "big1"); + assertEquals(0, cache.size()); + assertEquals(cache.get(junk20MB + "1"), null); + + // Do that again. + cache.put(junk20MB + "2", "big2"); + assertEquals(0, cache.size()); + assertEquals(cache.get(junk20MB + "2"), null); + + Files.deleteIfExists(fsPath); + } +} diff --git a/src/edu/stanford/nlp/sempre/corenlp/CoreNLPAnalyzer.java b/src/edu/stanford/nlp/sempre/corenlp/CoreNLPAnalyzer.java new file mode 100644 index 0000000..55ffffe --- /dev/null +++ b/src/edu/stanford/nlp/sempre/corenlp/CoreNLPAnalyzer.java @@ -0,0 +1,136 @@ +package edu.stanford.nlp.sempre.corenlp; + +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.LanguageInfo.DependencyEdge; + +import edu.stanford.nlp.ling.CoreAnnotations; +import edu.stanford.nlp.ling.IndexedWord; +import edu.stanford.nlp.ling.CoreAnnotations.*; +import edu.stanford.nlp.ling.CoreLabel; +import edu.stanford.nlp.pipeline.Annotation; +import edu.stanford.nlp.pipeline.StanfordCoreNLP; +import edu.stanford.nlp.semgraph.SemanticGraph; +import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations; +import edu.stanford.nlp.semgraph.SemanticGraphEdge; +import edu.stanford.nlp.util.CoreMap; +import com.google.common.collect.Lists; +import com.google.common.base.Joiner; +import fig.basic.*; +import java.util.*; + +/** + * CoreNLPAnalyzer uses Stanford CoreNLP pipeline to analyze an input string utterance + * and return a LanguageInfo object + * + * @author akchou + */ +public class CoreNLPAnalyzer extends LanguageAnalyzer { + public static class Options { + @Option(gloss = "What CoreNLP annotators to run") + public List annotators = Lists.newArrayList("tokenize", "ssplit", "pos", "lemma", "ner", "parse"); + + @Option(gloss = "Whether to use case-sensitive models") + public boolean caseSensitive = false; + } + + public static Options opts = new Options(); + + // TODO(pliang): don't muck with the POS tag; instead have a separate flag + // for isContent which looks at posTag != "MD" && lemma != "be" && lemma != + // "have" + // Need to update TextToTextMatcher + private static final String[] AUX_VERB_ARR = new String[] {"is", "are", "was", + "were", "am", "be", "been", "will", "shall", "have", "has", "had", + "would", "could", "should", "do", "does", "did", "can", "may", "might", + "must", "seem"}; + private static final Set AUX_VERBS = new HashSet(Arrays.asList(AUX_VERB_ARR)); + private static final String AUX_VERB_TAG = "VBD-AUX"; + + public static StanfordCoreNLP pipeline = null; + + public static void initModels() { + if (pipeline != null) return; + Properties props = new Properties(); + props.put("annotators", Joiner.on(',').join(opts.annotators)); + if (opts.caseSensitive) { + props.put("pos.model", "edu/stanford/nlp/models/pos-tagger/english-bidirectional/english-bidirectional-distsim.tagger"); + props.put("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz,edu/stanford/nlp/models/ner/english.conll.4class.distsim.crf.ser.gz"); + } else { + props.put("pos.model", "edu/stanford/nlp/models/pos-tagger/english-caseless-left3words-distsim.tagger"); + props.put("ner.model", "edu/stanford/nlp/models/ner/english.all.3class.caseless.distsim.crf.ser.gz,edu/stanford/nlp/models/ner/english.conll.4class.caseless.distsim.crf.ser.gz"); + } + pipeline = new StanfordCoreNLP(props); + } + + // Stanford tokenizer doesn't break hyphens. + // Replace hypens with spaces for utterances like + // "Spanish-speaking countries" but not for "2012-03-28". + public static String breakHyphens(String utterance) { + StringBuilder buf = new StringBuilder(utterance); + for (int i = 0; i < buf.length(); i++) { + if (buf.charAt(i) == '-' && (i + 1 < buf.length() && Character.isLetter(buf.charAt(i + 1)))) + buf.setCharAt(i, ' '); + } + return buf.toString(); + } + + public LanguageInfo analyze(String utterance) { + LanguageInfo languageInfo = new LanguageInfo(); + + // Clear these so that analyze can hypothetically be called + // multiple times. + languageInfo.tokens.clear(); + languageInfo.posTags.clear(); + languageInfo.nerTags.clear(); + languageInfo.nerValues.clear(); + languageInfo.lemmaTokens.clear(); + languageInfo.dependencyChildren.clear(); + + // Break hyphens + utterance = breakHyphens(utterance); + + // Run Stanford CoreNLP + initModels(); + Annotation annotation = pipeline.process(utterance); + + for (CoreLabel token : annotation.get(CoreAnnotations.TokensAnnotation.class)) { + String word = token.get(TextAnnotation.class); + String wordLower = word.toLowerCase(); + if (LanguageAnalyzer.opts.lowerCaseTokens) { + languageInfo.tokens.add(wordLower); + } else { + languageInfo.tokens.add(word); + } + languageInfo.posTags.add( + AUX_VERBS.contains(wordLower) ? + AUX_VERB_TAG : + token.get(PartOfSpeechAnnotation.class)); + languageInfo.nerTags.add(token.get(NamedEntityTagAnnotation.class)); + languageInfo.lemmaTokens.add(token.get(LemmaAnnotation.class)); + languageInfo.nerValues.add(token.get(NormalizedNamedEntityTagAnnotation.class)); + } + + // Fills in a stanford dependency graph for constructing a feature + for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { + SemanticGraph ccDeps = sentence.get(SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation.class); + int sentenceBegin = sentence.get(CoreAnnotations.TokenBeginAnnotation.class); + + // Iterate over all tokens and their dependencies + for (int sourceTokenIndex = sentenceBegin; + sourceTokenIndex < sentence.get(CoreAnnotations.TokenEndAnnotation.class); + sourceTokenIndex++) { + final ArrayList outgoing = new ArrayList(); + languageInfo.dependencyChildren.add(outgoing); + IndexedWord node = ccDeps.getNodeByIndexSafe(sourceTokenIndex - sentenceBegin + 1); // + 1 for ROOT + if (node != null) { + for (SemanticGraphEdge edge : ccDeps.outgoingEdgeList(node)) { + final String relation = edge.getRelation().toString(); + final int targetTokenIndex = sentenceBegin + edge.getTarget().index() - 1; + outgoing.add(new DependencyEdge(relation, targetTokenIndex)); + } + } + } + } + return languageInfo; + } +} diff --git a/src/edu/stanford/nlp/sempre/corenlp/build.xml b/src/edu/stanford/nlp/sempre/corenlp/build.xml new file mode 100644 index 0000000..b8cb705 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/corenlp/build.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/edu/stanford/nlp/sempre/corenlp/test/CoreNLPSemanticFnTest.java b/src/edu/stanford/nlp/sempre/corenlp/test/CoreNLPSemanticFnTest.java new file mode 100644 index 0000000..9e2e21f --- /dev/null +++ b/src/edu/stanford/nlp/sempre/corenlp/test/CoreNLPSemanticFnTest.java @@ -0,0 +1,92 @@ +package edu.stanford.nlp.sempre.corenlp.test; + +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.corenlp.CoreNLPAnalyzer; +import edu.stanford.nlp.sempre.test.TestUtils; +import fig.basic.LispTree; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.List; + +import static org.testng.AssertJUnit.assertEquals; + +/** + * Test SemanticFns that depend on CoreNLP (e.g., NumberFn on "one thousand") + * @author Percy Liang + */ +public class CoreNLPSemanticFnTest { + private static Formula F(String s) { return Formula.fromString(s); } + + void check(Formula target, DerivationStream derivations) { + if (!derivations.hasNext()) throw new RuntimeException("Expected 1 derivation, got " + derivations); + assertEquals(target, derivations.next().formula); + } + + void check(Formula target, String utterance, SemanticFn fn, List children) { + Example ex = TestUtils.makeSimpleExample(utterance); + check(target, fn.call(ex, new SemanticFn.CallInfo(null, 0, ex.numTokens(), Rule.nullRule, children))); + } + + void check(Formula target, String utterance, SemanticFn fn) { + List empty = Collections.emptyList(); + check(target, utterance, fn, empty); + } + + void checkNumDerivations(DerivationStream derivations, int num) { + assertEquals(num, derivations.estimatedSize()); + } + + Derivation D(Formula f) { + return (new Derivation.Builder()) + .formula(f) + .prob(1.0) + .createDerivation(); + } + + LispTree T(String str) { + return LispTree.proto.parseFromString(str); + } + + // TODO(chaganty): Test bridge fn - requires freebase (?) + // TODO(chaganty): Test context fn + + @Test public void dateFn() { + LanguageAnalyzer.setSingleton(new CoreNLPAnalyzer()); + check(F("(date 2013 8 7)"), "August 7, 2013", new DateFn()); + check(F("(date 1982 -1 -1)"), "1982", new DateFn()); + check(F("(date -1 6 4)"), "june 4", new DateFn()); + } + + @Test public void filterNerTagFn() { + LanguageAnalyzer.setSingleton(new CoreNLPAnalyzer()); + FilterNerSpanFn filter = new FilterNerSpanFn(); + filter.init(T("(FilterNerSpanFn token PERSON)")); + Derivation child = new Derivation.Builder().createDerivation(); + Example ex = TestUtils.makeSimpleExample("where is Obama"); + assertEquals(filter.call(ex, + new SemanticFn.CallInfo(null, 0, 1, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + assertEquals(filter.call(ex, + new SemanticFn.CallInfo(null, 1, 2, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + assertEquals(filter.call(ex, + new SemanticFn.CallInfo(null, 2, 3, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + } + + // TODO(chaganty): Test fuzzy match fn + // TODO(chaganty): Test identity fn + // TODO(chaganty): Test join fn + // TODO(chaganty): Test lexicon fn + // TODO(chaganty): Test merge fn + + @Test public void numberFn() { + LanguageAnalyzer.setSingleton(new CoreNLPAnalyzer()); + check(F("(number 35000)"), "thirty-five thousand", new NumberFn()); + } + + // TODO(chaganty): Test select fn + // TODO(chaganty): Test simple lexicon fn + +} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/BinaryLexicon.java b/src/edu/stanford/nlp/sempre/fbalignment/lexicons/BinaryLexicon.java deleted file mode 100644 index ab4477f..0000000 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/BinaryLexicon.java +++ /dev/null @@ -1,348 +0,0 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; - -import com.google.common.collect.Lists; -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.FbFormulasInfo; -import edu.stanford.nlp.sempre.FbFormulasInfo.BinaryFormulaInfo; -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.Json; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.BinaryLexicalEntry; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.LexiconValue; -import edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers.EntryNormalizer; -import edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers.IdentityNormalizer; -import edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers.PrepDropNormalizer; -import edu.stanford.nlp.util.CollectionUtils; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.Pair.BySecondReversePairComparator; -import edu.stanford.nlp.util.Triple; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Option; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.*; - -/** - * Lexicon for binary predicates - * @author jonathanberant - */ -public class BinaryLexicon { - - public static class Options { - @Option(gloss = "Number of results return by the lexicon") - public int maxEntries = 1000; - @Option(gloss = "Whether to drop the preposition when querying the lexicon") - public boolean prepDropNormalization = true; - @Option(gloss = "Path to binary lexicon files") - public List binaryLexiconFilesPath = Lists.newArrayList("lib/fb_data/6/binaryInfoStringAndAlignment.txt"); - @Option(gloss = "Whether to prune the lexicon") public boolean pruneLexicon = false; - @Option(gloss = "Number of entries to leave after pruning") - public int pruneBeamSize = 5; - - @Option(gloss = "Verbosity") public int verbose = 0; - @Option(gloss = "Whether to use Jaccard as the only alignment statistic") - public boolean useOnlyJaccard = false; - @Option(gloss = "Alignment score to sort by") - public String keyToSortBy = INTERSECTION; - } - - public static Options opts = new Options(); - - private EntryNormalizer lexiconLoadingNormalizer; - private FbFormulasInfo fbFormulasInfo; - - public static final String INTERSECTION = "Intersection_size_typed"; - public static final String NL_TYPED = "NL_typed_size"; - public static final String FB_TYPED = "FB_typed_size"; - - - Map, BinaryLexicalEntry>> lexemeToEntryMap = new HashMap, BinaryLexicalEntry>>(); - - public BinaryLexicon() throws IOException { - - fbFormulasInfo = FbFormulasInfo.getSingleton(); - - //if we omit prepositions then the lexicon normalizer does that, otherwise, it is a normalizer that does nothing - if (opts.prepDropNormalization) { - lexiconLoadingNormalizer = new PrepDropNormalizer(); // the alignment lexicon already contains stemmed stuff so just need to drop prepositions - } else { - lexiconLoadingNormalizer = new IdentityNormalizer(); - } - - for (String lexiconFile : opts.binaryLexiconFilesPath) { - uploadFile(lexiconFile); - } - - if (opts.pruneLexicon) { - LogInfo.begin_track("Pruning lexicon"); - LogInfo.log("Pruning with beam size: " + opts.pruneBeamSize); - pruneLexicon(opts.pruneBeamSize); - LogInfo.end_track("Pruning lexicon"); - } - } - - private void uploadFile(String lexiconFile) throws IOException { - - LogInfo.begin_track_printAll("Loading lexicon file " + lexiconFile); - for (String line : IOUtils.readLines(lexiconFile)) { - - LexiconValue lv = Json.readValueHard(line, LexiconValue.class); - String lexemeKey = lv.lexeme; - String normalizedLexemeKey = lexiconLoadingNormalizer.normalize(lexemeKey); - addEntryToMap(lexemeToEntryMap, lexemeKey, lv); - if (!lexemeKey.equals(normalizedLexemeKey)) { - addEntryToMap(lexemeToEntryMap, normalizedLexemeKey, lv); - } - } - LogInfo.log("Number of entries: " + lexemeToEntryMap.size()); - LogInfo.end_track(); - } - - private void addEntryToMap(Map, BinaryLexicalEntry>> lexemeToEntryMap, String mapKey, LexiconValue lv) { - Map, BinaryLexicalEntry> entries = lexemeToEntryMap.get(mapKey); - if (entries == null) { - lexemeToEntryMap.put(mapKey, entries = new HashMap, LexicalEntry.BinaryLexicalEntry>()); - } - addEntry(entries, lv, mapKey); - } - - public void addEntry(Map, BinaryLexicalEntry> entries, LexiconValue lexValue, String mapKey) { - - List binaryEntries = buildEntry(lexValue, mapKey); - for (BinaryLexicalEntry binaryEntry : binaryEntries) { - Pair formulaLexemePair = new Pair(binaryEntry.formula, binaryEntry.fullLexeme); - if (entries.containsKey(formulaLexemePair)) { - BinaryLexicalEntry otherEntry = entries.get(formulaLexemePair); - if (!binaryEntry.identicalFormulaInfo(otherEntry)) { - throw new RuntimeException("Different entries for same formula, existing entry: " + otherEntry + ", new entry: " + binaryEntry); - } - //hack - if (binaryEntry.source == EntrySource.STRING_MATCH && otherEntry.source == EntrySource.STRING_MATCH) - continue; - if (binaryEntry.source == EntrySource.ALIGNMENT && otherEntry.source == EntrySource.STRING_MATCH) { - otherEntry.alignmentScores = binaryEntry.alignmentScores; - otherEntry.source = binaryEntry.source; - continue; - } - if (binaryEntry.source == EntrySource.STRING_MATCH && otherEntry.source == EntrySource.ALIGNMENT) - continue; - for (String scoreDesc : binaryEntry.alignmentScores.keySet()) { - if (otherEntry.alignmentScores.containsKey(scoreDesc)) - throw new RuntimeException("While trying to merge entries found that both entries have the score: " + scoreDesc); - otherEntry.alignmentScores.put(scoreDesc, binaryEntry.alignmentScores.get(scoreDesc)); - } - } else { - entries.put(new Pair(binaryEntry.formula, binaryEntry.fullLexeme), binaryEntry); - } - } - } - - public List buildEntry(LexiconValue lexValue, String mapKey) { - - EntrySource source = EntrySource.parseSourceDesc(lexValue.source); - BinaryFormulaInfo info = fbFormulasInfo.getBinaryInfo(lexValue.formula); - - if(!validBinaryFormula(lexValue.formula)) - return Collections.emptyList(); - - if(info==null) { - if(opts.verbose>=3) - LogInfo.log("BinaryLexicon: skipping entry since there is no info for formula: " + lexValue.formula.toString()); - return Collections.emptyList(); - } - //get alignment features - Map alignmentScores = new TreeMap(lexValue.features); - - if (fbFormulasInfo.isCvtFormula(info) && source == EntrySource.STRING_MATCH) { - - List entries = new ArrayList(); - for (BinaryFormulaInfo cvtInfo : fbFormulasInfo.getCvtExpansions(info)) { - entries.add( - new BinaryLexicalEntry( - mapKey, mapKey, new HashSet(cvtInfo.descriptions), cvtInfo.formula, source, - cvtInfo.popularity, cvtInfo.expectedType1, cvtInfo.expectedType2, cvtInfo.unitId, cvtInfo.unitDesc, alignmentScores, lexValue.lexeme)); - } - return entries; - } else { - BinaryLexicalEntry entry = new BinaryLexicalEntry( - mapKey, mapKey, new HashSet(info.descriptions), lexValue.formula, source, - info.popularity, info.expectedType1, info.expectedType2, info.unitId, info.unitDesc, alignmentScores, lexValue.lexeme); - return Collections.singletonList(entry); - } - } - - public List lookupEntries(String textDesc) throws IOException { - - List res = new LinkedList(); - Map, BinaryLexicalEntry> entries = lexemeToEntryMap.get(textDesc.toLowerCase()); - - if (entries != null) { - for (BinaryLexicalEntry binaryEntry : entries.values()) { - if (opts.useOnlyJaccard) - binaryEntry.retainJaccardOnly(); - res.add(binaryEntry); - } - Collections.sort(res, new BinaryLexicalEntryComparator(opts.keyToSortBy)); - } - return res.subList(0, Math.min(res.size(), opts.maxEntries)); - } - - public void printLexiconStats() { - - Set fullLexemes = new HashSet(); - Set typedNomralizedLexemes = new HashSet(); - Set logicalForms = new HashSet(); - - for (String lexeme : lexemeToEntryMap.keySet()) { - for (BinaryLexicalEntry binaryEntry : lexemeToEntryMap.get(lexeme).values()) { - fullLexemes.add(binaryEntry.fullLexeme); - typedNomralizedLexemes.add(binaryEntry.normalizedTextDesc + ";" + binaryEntry.expectedType1 + ";" + binaryEntry.expectedType2); - logicalForms.add(binaryEntry.formula); - } - } - LogInfo.log("number of lexemes: " + lexemeToEntryMap.size()); - LogInfo.log("Number of full lexemes: " + fullLexemes.size()); - LogInfo.log("Number of typed normalized lexemes: " + typedNomralizedLexemes.size()); - LogInfo.log("Number of logical forms: " + logicalForms.size()); - } - - public void pruneLexicon(int topK) { - - for (String normLexeme : lexemeToEntryMap.keySet()) { - pruneLexicon(normLexeme, topK); - } - } - /** For every NL phrase choose the top-k matching ones */ - private void pruneLexicon(String normLexeme, int topK) { - - Map, BinaryLexicalEntry> entries = lexemeToEntryMap.get(normLexeme); - //sort by jaccard - List entryList = CollectionUtils.toList(entries.values()); - Collections.sort(entryList, new EntriesJaccardComparator()); - //keep the top 5 - List newEntries = new LinkedList(); - newEntries.addAll(entryList.subList(0, Math.min(entries.size(), topK))); - entries.clear(); - for (BinaryLexicalEntry newEntry : newEntries) - entries.put(new Pair(newEntry.formula, newEntry.fullLexeme), newEntry); - } - - - /** for every typed NL phrase choose the top-k */ - - public void pruneLexiconTyped(String normLexeme, int topK) { - - Map, BinaryLexicalEntry> entries = lexemeToEntryMap.get(normLexeme); - - //sort by triples - Map, List>> typedNlToScore = new HashMap, List>>(); - for (BinaryLexicalEntry entry : entries.values()) { - - Triple fullLexAndTypes = new Triple(entry.fullLexeme, entry.expectedType1, entry.expectedType2); - - List> typedEntries = typedNlToScore.get(fullLexAndTypes); - if (typedEntries == null) { - typedEntries = new LinkedList>(); - typedNlToScore.put(fullLexAndTypes, typedEntries); - } - typedEntries.add(new Pair(entry, MapUtils.getDouble(entry.alignmentScores, INTERSECTION, 0.0))); - } - - List newEntries = new LinkedList(); - - for (Triple typedNl : typedNlToScore.keySet()) { - - List> scoreList = typedNlToScore.get(typedNl); - Collections.sort(scoreList, new BySecondReversePairComparator()); - for (int i = 0; i < Math.min(scoreList.size(), topK); ++i) - newEntries.add(scoreList.get(i).first()); - } - entries.clear(); - for (BinaryLexicalEntry newEntry : newEntries) { - entries.put(new Pair(newEntry.formula, newEntry.fullLexeme), newEntry); - } - } - - public static class EntriesJaccardComparator implements Comparator { - - @Override - public int compare(BinaryLexicalEntry o1, BinaryLexicalEntry o2) { - - double smoothedJaccard1 = computeUntypedJaccardFromAlignmentScores(o1.alignmentScores); - double smoothedJaccard2 = computeUntypedJaccardFromAlignmentScores(o2.alignmentScores); - - if (smoothedJaccard1 > smoothedJaccard2) - return -1; - else if (smoothedJaccard1 < smoothedJaccard2) - return 1; - return 0; - } - - public static double computeUntypedJaccardFromAlignmentScores(Map alignmentScores) { - - double intersection = MapUtils.getDouble(alignmentScores, INTERSECTION, 0.0); - double nlSize = MapUtils.getDouble(alignmentScores, NL_TYPED, 0.0); - double fbSize = MapUtils.getDouble(alignmentScores, FB_TYPED, 0.0); - - double smoothedJaccard = intersection / (nlSize + fbSize - intersection + 5); - return smoothedJaccard; - } - } - - /** If the property has a reverse, keep it if it reversed*/ - public boolean validBinaryFormula(Formula formula) { - if(fbFormulasInfo.hasOpposite(formula)) { - boolean valid = fbFormulasInfo.isReversed(formula); - if(opts.verbose>=3) { - if(!valid) - LogInfo.logs("BinaryLexicon: invalid formula: %s",formula); - else - LogInfo.logs("BinaryLexicon: valid formula: %s",formula); - } - return valid; - } - return true; - } - - public void printLexemeAndFbDesc(String out) throws IOException { - PrintWriter writer = IOUtils.getPrintWriter(out); - - for (String lexeme : lexemeToEntryMap.keySet()) { - Map, BinaryLexicalEntry> entries = lexemeToEntryMap.get(lexeme); - for (Pair pair : entries.keySet()) { - BinaryLexicalEntry entry = entries.get(pair); - if (entry.jaccard() > 0.001) - writer.println(pair.second + "\t" + entry.fbDescriptions + "\t" + entry.jaccard()); - } - } - writer.close(); - } - - public static class BinaryLexicalEntryComparator implements Comparator { - - public String keyToSortby = null; - - public BinaryLexicalEntryComparator(String keyToSortBy) { - assert (keyToSortBy != null); - this.keyToSortby = keyToSortBy; - } - - @Override - public int compare(BinaryLexicalEntry arg0, BinaryLexicalEntry arg1) { - - double arg0Score = MapUtils.getDouble(arg0.alignmentScores, keyToSortby, 0.0); - double arg1Score = MapUtils.getDouble(arg1.alignmentScores, keyToSortby, 0.0); - - if (arg0Score > arg1Score) - return -1; - if (arg0Score < arg1Score) - return 1; - if (arg0.popularity > arg1.popularity) - return -1; - if (arg0.popularity < arg1.popularity) - return 1; - return 0; - } - } -} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/UnaryLexicon.java b/src/edu/stanford/nlp/sempre/fbalignment/lexicons/UnaryLexicon.java deleted file mode 100644 index 0e2e865..0000000 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/UnaryLexicon.java +++ /dev/null @@ -1,308 +0,0 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; - -import com.google.common.base.Strings; -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.FbFormulasInfo; -import edu.stanford.nlp.sempre.FbFormulasInfo.UnaryFormulaInfo; -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.Json; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.LexiconValue; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.UnaryLexicalEntry; -import edu.stanford.nlp.util.Pair; -import edu.stanford.nlp.util.Pair.BySecondReversePairComparator; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Option; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.*; - -public class UnaryLexicon { - - public static class Options { - @Option(gloss = "Number of results return by the lexicon") - public int maxEntries = 1000; - @Option(gloss = "Path to unary lexicon file") - public String unaryLexiconFilePath = "lib/fb_data/6/unaryInfoStringAndAlignment.txt"; - @Option(gloss = "Whether to prune the lexicon") public boolean pruneLexicon = false; - @Option(gloss = "Number of entries to leave after pruning") - public int pruneBeamSize = 5; - @Option(gloss = "Threshold for filtering unaries") - public int unaryFilterThreshold = 5; - @Option(gloss = "Verbosity") - public int verbose = 0; - - } - - public static Options opts = new Options(); - - private Map, UnaryLexicalEntry>> lexemeToEntryMap; - private static FbFormulasInfo fbFormulasInfo; - - public final static String INTERSECTION = "intersection"; - public final static String NL_SIZE = "nl_size"; - public final static String FB_SIZE = "fb_size"; - - public UnaryLexicon() { - - fbFormulasInfo = FbFormulasInfo.getSingleton(); - lexemeToEntryMap = new HashMap, UnaryLexicalEntry>>(); - - //get lexicon files names - List unaryLexiconFileNames = new LinkedList(); - if (!Strings.isNullOrEmpty(opts.unaryLexiconFilePath)) - unaryLexiconFileNames.add(opts.unaryLexiconFilePath); - - for (String lexiconFileName : unaryLexiconFileNames) { - loadLexiconFileIntoMap(lexiconFileName, lexemeToEntryMap); - } - - if (opts.pruneLexicon) { - LogInfo.begin_track("Pruning lexicon"); - System.out.println("Pruning with beam size: " + opts.pruneBeamSize); - pruneLexicon(opts.pruneBeamSize); - LogInfo.end_track("Pruning lexicon"); - } - } - - public void pruneLexicon(int topK) { - - for (String normLexeme : lexemeToEntryMap.keySet()) { - pruneLexicon(normLexeme, topK); - } - } - /** For every NL phrase choose the top-k matching ones */ - private void pruneLexicon(String normLexeme, int topK) { - - Map, UnaryLexicalEntry> entries = lexemeToEntryMap.get(normLexeme); - if (entries.size() <= topK) - return; // no need to prune - //sort by jaccard - List, Double>> formulaToScoreList = new LinkedList, Double>>(); - for (Pair formulaAndSource : entries.keySet()) { - UnaryLexicalEntry uEntry = entries.get(formulaAndSource); - double jaccard = computeJaccard(uEntry.alignmentScores); - formulaToScoreList.add(new Pair, Double>(formulaAndSource, jaccard)); - } - Collections.sort(formulaToScoreList, new BySecondReversePairComparator, Double>()); - - for (int i = topK; i < formulaToScoreList.size(); ++i) { - entries.remove(formulaToScoreList.get(i).first()); - } - } - - private double computeJaccard(Map alignmentScores) { - double intersection = MapUtils.getDouble(alignmentScores, INTERSECTION, 0.0); - double nlSize = MapUtils.getDouble(alignmentScores, NL_SIZE, 0.0); - double fbSize = MapUtils.getDouble(alignmentScores, FB_SIZE, 0.0); - - double smoothedJaccard = intersection / (nlSize + fbSize - intersection + 5); - return smoothedJaccard; - } - - private UnaryLexicon( - Map, UnaryLexicalEntry>> nlToFormulaAndAlignmentMap) { - this.lexemeToEntryMap = nlToFormulaAndAlignmentMap; - } - - public static UnaryLexicon fromUnaryLexiconFiles(List lexiconFileNames, int numOfResults) { - - Map, UnaryLexicalEntry>> nlToFormulaAndAlignmentMap = - new HashMap, UnaryLexicalEntry>>(); - - for (String lexiconFileName : lexiconFileNames) { - loadLexiconFileIntoMap(lexiconFileName, nlToFormulaAndAlignmentMap); - } - return new UnaryLexicon(nlToFormulaAndAlignmentMap); - } - - private static void loadLexiconFileIntoMap(String lexiconFileName, - Map, UnaryLexicalEntry>> nlToFormulaAndAlignmentMap) { - LogInfo.begin_track("Loading lexicon file " + lexiconFileName); - - for (String line : IOUtils.readLines(lexiconFileName)) { - LexiconValue lv = Json.readValueHard(line, LexiconValue.class); - addEntry(lv.lexeme, lv.source, lv.formula, lv.features, nlToFormulaAndAlignmentMap); - } - LogInfo.log("Number of entries: " + nlToFormulaAndAlignmentMap.size()); - LogInfo.end_track(); - } - - private static void addEntry(String nl, String source, Formula formula, Map featureMap, - Map, UnaryLexicalEntry>> nlToFormulaAndAlignmentMap) { - - if(fbFormulasInfo.getUnaryInfo(formula)==null) { - if(opts.verbose>=3) - LogInfo.log("Missing info for unary: " + formula); - } - else { - UnaryFormulaInfo uInfo = fbFormulasInfo.getUnaryInfo(formula); - UnaryLexicalEntry uEntry = new UnaryLexicalEntry(nl, nl, new TreeSet(uInfo.descriptions), formula, EntrySource.parseSourceDesc(source), - uInfo.popularity, new TreeMap(featureMap), uInfo.types); - - Map, UnaryLexicalEntry> formulaToAlignmentInfoMap = nlToFormulaAndAlignmentMap.get(nl); - if (formulaToAlignmentInfoMap == null) { - formulaToAlignmentInfoMap = new HashMap, UnaryLexicalEntry>(); - formulaToAlignmentInfoMap.put(new Pair(formula, source.toString()), uEntry); - nlToFormulaAndAlignmentMap.put(nl, formulaToAlignmentInfoMap); - } else { - //assumes same name and formula appear only once in the file - formulaToAlignmentInfoMap.put(new Pair(formula, source.toString()), uEntry); - } - } - } - - /** - * @param alignmentFile - file containing alignment from stemmed NL to - * Freebase unaries - * @param stringFile - file containing string info and popularity about all - * Freebase unaries - */ - public static UnaryLexicon fromStringFileAndAlignmentFile(String alignmentFile, String stringFile) { - - LogInfo.begin_track("Generating nl to info from string match file"); - Map, UnaryLexicalEntry>> nlToFormulaAndAlignmentMap = - generateFromStringFileNlToFormulaMap(stringFile); - LogInfo.end_track(); - LogInfo.begin_track("Adding alignment info"); - addAlignmentInfo(alignmentFile, nlToFormulaAndAlignmentMap); - LogInfo.end_track(); - return new UnaryLexicon(nlToFormulaAndAlignmentMap); - } - - public static void saveUnaryLexiconFromStringFileAndAlignmentFile(String alignmentFile, String stringFile, String lexiconFile) throws IOException { - UnaryLexicon lexicon = fromStringFileAndAlignmentFile(alignmentFile, stringFile); - lexicon.saveUnaryLexiconFile(lexiconFile); - } - - private static void addAlignmentInfo(String alignmentFile, - Map, UnaryLexicalEntry>> nlToFormulaAndAlignmentMap) { - - int i = 0; - for (String line : IOUtils.readLines(alignmentFile)) { - i++; - if (i == 1) - continue; - String[] tokens = line.split("\t"); - String nl = tokens[0]; //it is already stemmed and normalized - Formula formula = Formula.fromString(tokens[1]); - Map features = new TreeMap(); - features.put(INTERSECTION, Double.parseDouble(tokens[2])); - features.put(NL_SIZE, Double.parseDouble(tokens[3])); - features.put(FB_SIZE, Double.parseDouble(tokens[4])); - - addEntry(nl, EntrySource.ALIGNMENT.toString(), formula, features, nlToFormulaAndAlignmentMap); - if (i % 10000 == 0) - LogInfo.log("Number of lines: " + i); - } - } - - private static Map, UnaryLexicalEntry>> generateFromStringFileNlToFormulaMap(String stringFile) { - - Map, UnaryLexicalEntry>> res = - new HashMap, UnaryLexicalEntry>>(); - - int i = 0; - for (String line : IOUtils.readLines(stringFile)) { - - String[] tokens = line.split("\t"); - String nl = tokens[3].toLowerCase(); - Formula formula = Formula.fromString(tokens[1]); - if(fbFormulasInfo.getUnaryInfo(formula)==null) { - LogInfo.log("Skipping on formula since info is missing from schema: " + formula); - } - else { - UnaryFormulaInfo uInfo = fbFormulasInfo.getUnaryInfo(formula); - - Map, UnaryLexicalEntry> formulaToAlignmentInfoMap = res.get(nl); - if (formulaToAlignmentInfoMap == null) { - formulaToAlignmentInfoMap = new HashMap, UnaryLexicalEntry>(); - res.put(nl, formulaToAlignmentInfoMap); - } - UnaryLexicalEntry uEntry = - formulaToAlignmentInfoMap.get(new Pair(formula,EntrySource.STRING_MATCH.toString())); - if (uEntry == null) { - - uEntry = new UnaryLexicalEntry(nl, nl, new TreeSet(uInfo.descriptions), formula, EntrySource.STRING_MATCH, uInfo.popularity, - new TreeMap(), uInfo.types); - formulaToAlignmentInfoMap.put(new Pair(formula, EntrySource.STRING_MATCH.toString()), uEntry); - } - i++; - if (i % 1000 == 0) - LogInfo.log("Adding mapping from nl: " + nl + " to formula " + formula.toString()); - } - } - return res; - } - - public void saveUnaryLexiconFile(String outFile) throws IOException { - - PrintWriter writer = IOUtils.getPrintWriter(outFile); - for (String nl : lexemeToEntryMap.keySet()) { - for (Pair formulaAndSource : lexemeToEntryMap.get(nl).keySet()) { - UnaryLexicalEntry uEntry = lexemeToEntryMap.get(nl).get(formulaAndSource); - LexiconValue lv = new LexiconValue(nl, formulaAndSource.first(), formulaAndSource.second(), uEntry.alignmentScores); - writer.println(Json.writeValueAsStringHard(lv)); - } - } - writer.close(); - } - - public List lookupEntries(String textDesc) throws IOException { - - List res = new LinkedList(); - - String normalizedTextDesc = textDesc.toLowerCase(); - Map, UnaryLexicalEntry> entries = lexemeToEntryMap.get(normalizedTextDesc); - - if (entries != null) { - for (Pair formulaAndSource : entries.keySet()) { - UnaryLexicalEntry uEntry = entries.get(formulaAndSource); - if (valid(uEntry)) - res.add(uEntry); - } - Collections.sort(res, new UnaryLexicalEntryComparator()); - } - return res.subList(0, Math.min(res.size(), opts.maxEntries)); - } - - /** Checks if an entry is valid (e.g. we filter if intersection is too small) */ - private boolean valid(UnaryLexicalEntry lexicalEntry) { - return (lexicalEntry.source != EntrySource.ALIGNMENT || - MapUtils.getDouble(lexicalEntry.alignmentScores, INTERSECTION, 0.0) >= opts.unaryFilterThreshold); - } - - public void printStats() { - - Set formulaSet = new HashSet(); - for (String nl : lexemeToEntryMap.keySet()) { - for (Pair formulaAndSource : lexemeToEntryMap.get(nl).keySet()) { - formulaSet.add(formulaAndSource.first()); - } - } - LogInfo.log("number of phrases: " + lexemeToEntryMap.size()); - LogInfo.log("Number fo formulas: " + formulaSet.size()); - } - - public static class UnaryLexicalEntryComparator implements Comparator { - - public static final String INTERSECTION = "intersection"; - - @Override - public int compare(UnaryLexicalEntry arg0, UnaryLexicalEntry arg1) { - - double arg0Intersection = MapUtils.getDouble(arg0.alignmentScores, INTERSECTION,0.0); - double arg1Intersection = MapUtils.getDouble(arg1.alignmentScores, INTERSECTION,0.0); - if (arg0Intersection > arg1Intersection) - return -1; - if (arg0Intersection < arg1Intersection) - return 1; - if (arg0.popularity > arg1.popularity) - return -1; - if (arg0.popularity < arg1.popularity) - return 1; - return 0; - } - } -} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/WordDistance.java b/src/edu/stanford/nlp/sempre/fbalignment/lexicons/WordDistance.java deleted file mode 100644 index 0288208..0000000 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/WordDistance.java +++ /dev/null @@ -1,77 +0,0 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; - -import com.google.common.base.Strings; -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.fbalignment.utils.MathUtils; -import fig.basic.Option; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * We define similarity to be positive - * - * @author jonathanberant - */ -public class WordDistance { - - private Map> wordVectors; - - public static class Options { - @Option(gloss = "Path to file containing word vectors, one per line") public String wordVectorFile; - @Option(gloss = "Method for calculuating distances") public String distanceMethod = "cosine"; - } - - public static Options opts = new Options(); - - private static WordDistance wordDistance; - public static WordDistance getSingleton() { - if (wordDistance == null) wordDistance = new WordDistance(); - return wordDistance; - } - - private WordDistance() { - wordVectors = new HashMap>(); - - if (Strings.isNullOrEmpty(opts.wordVectorFile)) - return; - - for (String line : IOUtils.readLines(opts.wordVectorFile)) { - String[] tokens = line.split("\\s+"); - List vector = new ArrayList(); - for (int i = 1; i < tokens.length; ++i) - vector.add(Double.parseDouble(tokens[i])); - wordVectors.put(tokens[0], vector); - } - } - - public double score(String word1, String word2) { - if (!wordVectors.containsKey(word1) || !wordVectors.containsKey(word2)) - return noVectorScore(); - if (opts.distanceMethod.equals("cosine")) { - return Math.max(MathUtils.vectorCosine(wordVectors.get(word1), wordVectors.get(word2)), 0); - } - if (opts.distanceMethod.equals("euclid")) { - return MathUtils.euclidDistance(wordVectors.get(word1), wordVectors.get(word2)); - } - throw new RuntimeException("Unknown similarity method: " + opts.distanceMethod); - } - - private double noVectorScore() { - if (opts.distanceMethod.equals("cosine")) - return 0.0; - if (opts.distanceMethod.equals("euclid")) - return 10.0; - throw new RuntimeException("Unknown similarity method: " + opts.distanceMethod); - } - - public static ExtremeValueWrapper buildDistanceWrapper() { - if (opts.distanceMethod.equals("cosine")) - return new MaxValueWrapper(0); - if (opts.distanceMethod.equals("euclid")) - return new MinValueWrapper(10); - throw new RuntimeException("Unknown similarity method: " + opts.distanceMethod); - } -} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/EntryNormalizer.java b/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/EntryNormalizer.java deleted file mode 100644 index f83a472..0000000 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/EntryNormalizer.java +++ /dev/null @@ -1,7 +0,0 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers; - -public interface EntryNormalizer { - - public String normalize(String str); - -} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/utils/CollectionUtils.java b/src/edu/stanford/nlp/sempre/fbalignment/utils/CollectionUtils.java deleted file mode 100644 index 7b17c67..0000000 --- a/src/edu/stanford/nlp/sempre/fbalignment/utils/CollectionUtils.java +++ /dev/null @@ -1,25 +0,0 @@ -package edu.stanford.nlp.sempre.fbalignment.utils; - -import java.util.HashMap; -import java.util.Map; - - -public class CollectionUtils { - - public static Map arraysToMap(K[] keys, V[] values) { - if(keys.length!=values.length) - throw new RuntimeException("Lenght of keys: " + keys.length+", length of values: " + values.length); - Map res = new HashMap<>(); - for(int i = 0; i < keys.length; ++i) { - res.put(keys[i], values[i]); - } - return res; - } - - public static Map doubleContainerToDoubleMap(Map map) { - Map res = new HashMap<>(); - for(K key: map.keySet()) - res.put(key, map.get(key).value()); - return res; - } -} diff --git a/src/edu/stanford/nlp/sempre/freebase/BinaryLexicon.java b/src/edu/stanford/nlp/sempre/freebase/BinaryLexicon.java new file mode 100644 index 0000000..c31d6c5 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/BinaryLexicon.java @@ -0,0 +1,282 @@ +package edu.stanford.nlp.sempre.freebase; + +import com.google.common.base.Strings; +import edu.stanford.nlp.io.IOUtils; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.BinaryFormulaInfo; +import edu.stanford.nlp.sempre.freebase.lexicons.EntrySource; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.BinaryLexicalEntry; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.LexiconValue; +import edu.stanford.nlp.sempre.freebase.lexicons.normalizers.EntryNormalizer; +import edu.stanford.nlp.sempre.freebase.lexicons.normalizers.PrepDropNormalizer; +import fig.basic.*; +import fig.exec.Execution; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +/** + * Lexicon for binary predicates, "born" --> fb:people.person.place_of_birth + * @author jonathanberant + */ +public final class BinaryLexicon { + + public static class Options { + @Option(gloss = "Number of results return by the lexicon") + public int maxEntries = 1000; + @Option(gloss = "Path to binary lexicon files") + public String binaryLexiconFilesPath = "lib/fb_data/7/binaryInfoStringAndAlignment.txt"; + @Option(gloss = "Verbosity") public int verbose = 0; + @Option(gloss = "Alignment score to sort by") + public String keyToSortBy = INTERSECTION; + } + + private static BinaryLexicon binaryLexicon; + public static BinaryLexicon getInstance() { + if (binaryLexicon == null) + try { + binaryLexicon = new BinaryLexicon(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return binaryLexicon; + } + + public static Options opts = new Options(); + + private EntryNormalizer lexiconLoadingNormalizer; + private FbFormulasInfo fbFormulasInfo; + + public static final String INTERSECTION = "Intersection_size_typed"; + + Map> lexemeToEntryList = new HashMap<>(); + + private BinaryLexicon() throws IOException { + if (Strings.isNullOrEmpty(opts.binaryLexiconFilesPath)) + throw new RuntimeException("Missing unary lexicon file"); + fbFormulasInfo = FbFormulasInfo.getSingleton(); + // if we omit prepositions then the lexicon normalizer does that, otherwise, it is a normalizer that does nothing + lexiconLoadingNormalizer = new PrepDropNormalizer(); // the alignment lexicon already contains stemmed stuff so just need to drop prepositions + read(opts.binaryLexiconFilesPath); + } + + private void read(String lexiconFile) throws IOException { + + LogInfo.begin_track_printAll("Loading binary lexicon file " + lexiconFile); + for (String line : IOUtils.readLines(lexiconFile)) { + LexiconValue lv = Json.readValueHard(line, LexiconValue.class); + String lexemeKey = lv.lexeme; + String normalizedLexemeKey = lexiconLoadingNormalizer.normalize(lexemeKey); + // add lexeme and normalized lexeme + addEntryToMap(lexemeKey, lv); + if (!lexemeKey.equals(normalizedLexemeKey)) { + addEntryToMap(normalizedLexemeKey, lv); + } + } + sortLexiconEntries(); + LogInfo.log("Number of entries: " + lexemeToEntryList.size()); + LogInfo.end_track(); + } + + public void addEntryToMap(String lexemeKey, LexiconValue lv) { + List bEntries = buildEntry(lv, lexemeKey); + for (BinaryLexicalEntry bEntry : bEntries) + MapUtils.addToList(lexemeToEntryList, lexemeKey, bEntry); + } + + private void sortLexiconEntries() { + for (List entries: lexemeToEntryList.values()) { + Collections.sort(entries, new BinaryLexEntryByCounterComparator()); + } + } + + public List buildEntry(LexiconValue lexValue, String lexemeKey) { + + EntrySource source = EntrySource.parseSourceDesc(lexValue.source); + BinaryFormulaInfo info = fbFormulasInfo.getBinaryInfo(lexValue.formula); + + if (!validBinaryFormula(lexValue.formula)) + return Collections.emptyList(); + + if (info == null) { + if (opts.verbose >= 3) + LogInfo.log("BinaryLexicon: skipping entry since there is no info for formula: " + lexValue.formula.toString()); + return Collections.emptyList(); + } + // get alignment features + Map alignmentScores = new TreeMap<>(lexValue.features); + + if (fbFormulasInfo.isCvtFormula(info) && source == EntrySource.STRING_MATCH) { + + List entries = new ArrayList<>(); + for (BinaryFormulaInfo cvtInfo : fbFormulasInfo.getCvtExpansions(info)) { + entries.add( + new BinaryLexicalEntry( + lexemeKey, lexemeKey, new HashSet<>(cvtInfo.descriptions), cvtInfo.formula, source, + cvtInfo.popularity, cvtInfo.expectedType1, cvtInfo.expectedType2, cvtInfo.unitId, cvtInfo.unitDesc, alignmentScores, lexValue.lexeme)); + } + return entries; + } else { + BinaryLexicalEntry entry = new BinaryLexicalEntry( + lexemeKey, lexemeKey, new HashSet<>(info.descriptions), lexValue.formula, source, + info.popularity, info.expectedType1, info.expectedType2, info.unitId, info.unitDesc, alignmentScores, lexValue.lexeme); + return Collections.singletonList(entry); + } + } + + public List lookupEntries(String textDesc) throws IOException { + List entries = lexemeToEntryList.get(textDesc.toLowerCase()); + if (entries != null) { + List res = new ArrayList<>(); + for (int i = 0; i < Math.min(entries.size(), opts.maxEntries); ++i) { + res.add(entries.get(i)); + } + return res; + } + return Collections.emptyList(); + } + + /** If the property has a reverse, keep it if it reversed*/ + public boolean validBinaryFormula(Formula formula) { + if (fbFormulasInfo.hasOpposite(formula)) { + boolean valid = fbFormulasInfo.isReversed(formula); + if (opts.verbose >= 3) { + if (!valid) + LogInfo.logs("BinaryLexicon: invalid formula: %s", formula); + else + LogInfo.logs("BinaryLexicon: valid formula: %s", formula); + } + return valid; + } + return true; + } + + public void updateLexicon(Pair lexemeFormulaPair, int support) { + StopWatchSet.begin("BinaryLexicon.updateLexicon"); + if (opts.verbose > 0) + LogInfo.logs("Pair=%s, score=%s", lexemeFormulaPair, support); + boolean exists = false; + String lexeme = lexemeFormulaPair.getFirst(); + Formula formula = lexemeFormulaPair.getSecond(); + + List bEntries = MapUtils.get(lexemeToEntryList, lexeme, Collections.emptyList()); + for (BinaryLexicalEntry bEntry : bEntries) { + if (bEntry.formula.equals(formula)) { + bEntry.alignmentScores.put("Feedback", (double) support); + if (opts.verbose > 0) + LogInfo.logs("Entry exists: %s", bEntry); + exists = true; + break; + } + } + if (!exists) { + BinaryFormulaInfo bInfo = fbFormulasInfo.getBinaryInfo(formula); + if (bInfo == null) { + LogInfo.warnings("BinaryLexicon.updateLexicon: no binary info for %s", formula); + return; + } + BinaryLexicalEntry newEntry = + new BinaryLexicalEntry( + lexeme, lexeme, new HashSet<>(bInfo.descriptions), bInfo.formula, EntrySource.FEEDBACK, + bInfo.popularity, bInfo.expectedType1, bInfo.expectedType2, bInfo.unitId, bInfo.unitDesc, new HashMap<>(), lexeme); + MapUtils.addToList(lexemeToEntryList, lexeme, newEntry); + newEntry.alignmentScores.put("Feedback", (double) support); + LogInfo.logs("Adding new binary entry=%s", newEntry); + + } + StopWatchSet.end(); + } + + public void sortLexiconByFeedback(Params params) { + StopWatchSet.begin("BinaryLexicon.sortLexiconByFeedback"); + LogInfo.log("Number of entries: " + lexemeToEntryList.size()); + BinaryLexEntrybyFeaturesComparator comparator = + new BinaryLexEntrybyFeaturesComparator(params); + for (String lexeme : lexemeToEntryList.keySet()) { + Collections.sort(lexemeToEntryList.get(lexeme), comparator); + if (opts.verbose > 1) { + LogInfo.logs("Sorted list for lexeme=%s", lexeme); + for (BinaryLexicalEntry bEntry : lexemeToEntryList.get(lexeme)) { + FeatureVector fv = new FeatureVector(); + LexiconFn.getBinaryEntryFeatures(bEntry, fv); + LogInfo.logs("Entry=%s, dotprod=%s", bEntry, fv.dotProduct(comparator.params)); + } + } + } + try { + // Output the lexicon to the execution directory. + String path = Execution.getFile("lexicon"); + if (path != null) { + PrintWriter writer = fig.basic.IOUtils.openOut(path); + for (String lexeme : lexemeToEntryList.keySet()) { + writer.println(lexeme + "\t" + lexemeToEntryList.get(lexeme)); + } + writer.flush(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + + StopWatchSet.end(); + } + + public class BinaryLexEntrybyFeaturesComparator implements Comparator { + + public final Params params; + public BinaryLexEntrybyFeaturesComparator(Params params) { + this.params = params; + } + @Override + public int compare(BinaryLexicalEntry entry1, BinaryLexicalEntry entry2) { + + FeatureVector features1 = new FeatureVector(); + FeatureVector features2 = new FeatureVector(); + LexiconFn.getBinaryEntryFeatures(entry1, features1); + LexiconFn.getBinaryEntryFeatures(entry2, features2); + double score1 = features1.dotProduct(params); + double score2 = features2.dotProduct(params); + if (score1 > score2) return -1; + if (score1 < score2) return +1; + // back off to usual thing + double entry1Score = MapUtils.getDouble(entry1.alignmentScores, opts.keyToSortBy, 0.0); + double entry2Score = MapUtils.getDouble(entry2.alignmentScores, opts.keyToSortBy, 0.0); + + if (entry1Score > entry2Score) + return -1; + if (entry1Score < entry2Score) + return +1; + if (entry1.popularity > entry2.popularity) + return -1; + if (entry1.popularity < entry2.popularity) + return +1; + return 0; + } + } + + public class BinaryLexEntryByCounterComparator implements Comparator { + + @Override + public int compare(BinaryLexicalEntry entry1, BinaryLexicalEntry entry2) { + double entry1Score = MapUtils.getDouble(entry1.alignmentScores, opts.keyToSortBy, 0.0); + double entry2Score = MapUtils.getDouble(entry2.alignmentScores, opts.keyToSortBy, 0.0); + + if (entry1Score > entry2Score) + return -1; + if (entry1Score < entry2Score) + return +1; + if (entry1.popularity > entry2.popularity) + return -1; + if (entry1.popularity < entry2.popularity) + return +1; + // to do - this is to break ties - make more efficient + int stringComparison = entry1.formula.toString().compareTo(entry2.formula.toString()); + if (stringComparison < 0) + return -1; + if (stringComparison > 0) + return +1; + return 0; + } + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/BridgeFn.java b/src/edu/stanford/nlp/sempre/freebase/BridgeFn.java new file mode 100644 index 0000000..e05a48d --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/BridgeFn.java @@ -0,0 +1,548 @@ +package edu.stanford.nlp.sempre.freebase; + +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.BinaryFormulaInfo; +import edu.stanford.nlp.sempre.MergeFormula.Mode; +import fig.basic.LispTree; +import fig.basic.LogInfo; +import fig.basic.Option; +import edu.stanford.nlp.sempre.LanguageInfo.DependencyEdge; +import edu.stanford.nlp.sempre.*; + +import java.io.IOException; +import java.util.*; + +/** + * Bridge between two derivations by type-raising one of them. + * @author jonathanberant + */ +public class BridgeFn extends SemanticFn { + + private static final Formula intFormula = Formulas.fromLispTree(LispTree.proto.parseFromString("(fb:type.object.type fb:type.int)")); + private static final Formula floatFormula = Formulas.fromLispTree(LispTree.proto.parseFromString("(fb:type.object.type fb:type.float)")); + + public static class Options { + @Option (gloss = "Verbose") public int verbose = 0; + @Option (gloss = "Whether to have binary predicate features (ovrefits on small data)") + public boolean useBinaryPredicateFeatures = true; + @Option (gloss = "Whether to filter bad domains such as user and common") + public boolean filterBadDomain = true; + } + + public static Options opts = new Options(); + + private FbFormulasInfo fbFormulaInfo = null; + private String description; + private boolean headFirst; + private TextToTextMatcher textToTextMatcher; + + public void init(LispTree tree) { + super.init(tree); + if (tree.children.size() != 3) + throw new RuntimeException("Number of children is: " + tree.children.size()); + if (!tree.child(2).value.equals("headFirst") && !tree.child(2).value.equals("headLast")) + throw new RuntimeException("Bad argument for head position: " + tree.child(2).value); + if (!tree.child(1).value.equals("unary") && !tree.child(1).value.equals("inject") && !tree.child(1).value.equals("entity")) + throw new RuntimeException("Bad description: " + tree.child(1).value); + + this.description = tree.child(1).value; + headFirst = tree.child(2).value.equals("headFirst"); + } + + public BridgeFn() { + fbFormulaInfo = FbFormulasInfo.getSingleton(); + textToTextMatcher = TextToTextMatcher.getSingleton(); + } + + @Override + public DerivationStream call(Example ex, Callable c) { + try { + switch (description) { + case "unary": + return bridgeUnary(ex, c); + case "inject": + return injectIntoCvt(ex, c); + case "entity": + return bridgeEntity(ex, c); + default: + throw new RuntimeException("Invalid (expected unary, inject, or entity): " + description); + } + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + @Override + public void sortOnFeedback(Params params) { + LogInfo.begin_track("Learner.BridgeFeedback"); + FbFormulasInfo fbFormulasInfo = FbFormulasInfo.getSingleton(); + Comparator feedbackComparator = fbFormulasInfo.new FormulaByFeaturesComparator(params); + fbFormulasInfo.sortType2ToBinaryMaps(feedbackComparator); + LogInfo.end_track(); + } + + + private boolean isCvt(Derivation headDeriv) { + if (!(headDeriv.formula instanceof JoinFormula)) + return false; + JoinFormula join = (JoinFormula) headDeriv.formula; + return join.relation instanceof LambdaFormula || + join.child instanceof JoinFormula || join.child instanceof MergeFormula; + } + + // Return all the entity supertypes of |type|. + // TODO(joberant): make this more efficient. + private Set getSupertypes(SemType type, Set supertypes) { + if (type instanceof AtomicSemType) + supertypes.addAll(SemTypeHierarchy.singleton.getSupertypes(((AtomicSemType) type).name)); + else if (type instanceof UnionSemType) + for (SemType baseType : ((UnionSemType) type).baseTypes) + getSupertypes(baseType, supertypes); + else { + // TODO(joberant): FIXME HACK for when passing binary into lambda formula and + // getSuperTypes doesn't work + getSupertypes(SemType.fromString("topic"), supertypes); + // throw new RuntimeException("Unexpected type (must be unary): " + type); + } + return supertypes; + } + + private DerivationStream bridgeUnary(Example ex, Callable c) throws IOException { + + assert ex != null; + // Example (headFirst = false): modifier[Hanks] head[movies] + Derivation headDeriv = headFirst ? c.child(0) : c.child(1); + Derivation modifierDeriv = !headFirst ? c.child(0) : c.child(1); + + Set headTypes = getSupertypes(headDeriv.type, new HashSet<>()); + Set modifierTypes = getSupertypes(modifierDeriv.type, new HashSet<>()); + ArrayList bridgingInfoList = new ArrayList<>(); + + for (String modifierType : modifierTypes) { // For each head type... + List binaries = fbFormulaInfo.getBinariesForType2(modifierType); + for (Formula binary : binaries) { // For each possible binary... + if (opts.filterBadDomain && badDomain(binary)) + continue; + BinaryFormulaInfo binaryInfo = fbFormulaInfo.getBinaryInfo(binary); + + if (opts.verbose >= 3) + LogInfo.logs("%s => %s", modifierType, binary); + + if (headTypes.contains(binaryInfo.expectedType1)) { + BridgingInfo bridgingInfo = new BridgingInfo(ex, c, binaryInfo, headFirst, headDeriv, modifierDeriv); + bridgingInfoList.add(bridgingInfo); + } + } + } + Collections.sort(bridgingInfoList); + return new LazyBridgeFnDerivs(bridgingInfoList); + } + + // bridge without a unary - simply by looking at binaries leading to the entity and string matching binary description to example tokens/lemmas/stems + private DerivationStream bridgeEntity(Example ex, Callable c) throws IOException { + + assert ex != null; + Derivation modifierDeriv = c.child(0); + Set modifierTypes = getSupertypes(modifierDeriv.type, new HashSet<>()); + ArrayList bridgingInfoList = new ArrayList<>(); + + if (opts.verbose >= 1) + LogInfo.logs("bridgeEntity: %s | %s", modifierDeriv, modifierTypes); + + for (String modifierType : modifierTypes) { // For each head type... + List binaries = fbFormulaInfo.getBinariesForType2(modifierType); + for (Formula binary : binaries) { // For each possible binary... + if (opts.filterBadDomain && badDomain(binary)) + continue; + BinaryFormulaInfo binaryInfo = fbFormulaInfo.getBinaryInfo(binary); + + if (opts.verbose >= 3) + LogInfo.logs("%s => %s", modifierType, binary); + + BridgingInfo bridgingInfo = new BridgingInfo(ex, c, binaryInfo, headFirst, null, modifierDeriv); + bridgingInfoList.add(bridgingInfo); + } + } + Collections.sort(bridgingInfoList); + return new LazyBridgeFnDerivs(bridgingInfoList); + } + + private boolean badDomain(String binary) { + return binary.contains("fb:user.") || binary.contains("fb:base.") || binary.contains("fb:dataworld.") || + binary.contains("fb:type.") || binary.contains("fb:common.") || binary.contains("fb:freebase."); + } + + private boolean badDomain(Formula formula) { + if (formula instanceof VariableFormula) return false; + if (formula instanceof ValueFormula) { + return badDomain(formula.toString()); + } + if (formula instanceof JoinFormula) { + JoinFormula jFormula = (JoinFormula) formula; + return badDomain(jFormula.relation) || badDomain(jFormula.child); + } + if (formula instanceof LambdaFormula) { + LambdaFormula lambdaFormula = (LambdaFormula) formula; + return badDomain(lambdaFormula.body); + } + if (formula instanceof ReverseFormula) { + ReverseFormula reverseFormula = (ReverseFormula) formula; + return badDomain(reverseFormula.child); + } + if (formula instanceof NotFormula) { + NotFormula notFormula = (NotFormula) formula; + return badDomain(notFormula.child); + } + throw new RuntimeException("Binary has formula type that is not supported"); + } + + // generate from example array of content word tokens/lemmas/stems that are not dominated by child derivations + private List> generateExampleInfo(Example ex, Callable c) { + + List tokens = new ArrayList<>(); + List posTags = new ArrayList<>(); + List lemmas = new ArrayList<>(); + List> res = new ArrayList<>(); + res.add(tokens); + res.add(posTags); + res.add(lemmas); + + Derivation modifierDeriv = headFirst ? c.child(1) : c.child(0); + + for (int i = 0; i < ex.languageInfo.tokens.size(); ++i) { + if (i >= modifierDeriv.start && i < modifierDeriv.end) { // do not consider the modifier words { + continue; + } + tokens.add(ex.languageInfo.tokens.get(i)); + posTags.add(ex.languageInfo.posTags.get(i)); + lemmas.add(ex.languageInfo.lemmaTokens.get(i)); + } + return res; + } + + private DerivationStream injectIntoCvt(Example ex, Callable c) { + assert ex != null; + + if (opts.verbose >= 2) + LogInfo.logs("child1=%s, child2=%s", ex.phrase(c.child(0).start, c.child(0).end), ex.phrase(c.child(1).start, c.child(1).end)); + + // Example: modifier[Braveheart] head[Mel Gibson plays in] + Derivation headDeriv = headFirst ? c.child(0) : c.child(1); + if (!isCvt(headDeriv)) // only works on cvts + return new LazyBridgeFnDerivs(new ArrayList<>()); + Derivation modifierDeriv = !headFirst ? c.child(0) : c.child(1); + JoinFormula headFormula = (JoinFormula) Formulas.betaReduction(headDeriv.formula); + // find the type of the cvt node + Set headTypes = Collections.singleton(fbFormulaInfo.getBinaryInfo(headFormula.relation).expectedType2); + Set modifierTypes = getSupertypes(modifierDeriv.type, new HashSet<>()); + ArrayList bridgingInfoList = new ArrayList<>(); + + for (String modifierType : modifierTypes) { + List binaries = fbFormulaInfo.getAtomicBinariesForType2(modifierType); // here we use atomic binaries since we inject into a CVT + for (Formula binary : binaries) { // For each possible binary... + if (opts.filterBadDomain && badDomain(binary)) + continue; + BinaryFormulaInfo info = fbFormulaInfo.getBinaryInfo(binary); + + if (headTypes.contains(info.expectedType1)) { + BridgingInfo bridgingInfo = new BridgingInfo(ex, c, info, headFirst, headDeriv, modifierDeriv); + bridgingInfoList.add(bridgingInfo); + } + } + } + Collections.sort(bridgingInfoList); + return new LazyBridgeFnDerivs(bridgingInfoList); + } + + // Checks whether "var" is used as a binary in "formula" + private boolean varIsBinary(Formula formula, String var) { + boolean isBinary = false; + LispTree tree = formula.toLispTree(); + VariableFormula vf = new VariableFormula(var); + for (LispTree child : tree.children) { + if (child.isLeaf()) + continue; + if (child.children.size() == 2 && vf.equals(Formulas.fromLispTree(child.child(0)))) { + isBinary = true; + break; + } + if (varIsBinary(Formulas.fromLispTree(child), var)) { + isBinary = true; + break; + } + } + return isBinary; + } + + private Formula buildBridge(Formula headFormula, Formula modifierFormula, Formula binary) { + // Handle cases like "what state has the most cities" where "has the" is mapped + // to "contains" predicate via bridging but "most" triggers a nested lambda w/ + // argmax on a count relation + // (Corresponds to $MetaMetaOperator in grammar) + if (modifierFormula instanceof LambdaFormula) { + LambdaFormula lf = (LambdaFormula) modifierFormula; + if (varIsBinary(lf, lf.var)) { + Formula newBinary = Formulas.lambdaApply(lf, binary); + if (newBinary instanceof LambdaFormula) { + return Formulas.lambdaApply((LambdaFormula) newBinary, headFormula); + } + } + } + + Formula join = new JoinFormula(binary, modifierFormula); + Formula merge = new MergeFormula(Mode.and, headFormula, join); + // Don't merge on ints and floats + return (headFormula.equals(intFormula) || headFormula.equals(floatFormula)) ? join : merge; + } + + private Formula buildBridgeFromCvt(JoinFormula headFormula, Formula modifierFormula, Formula binary) { + Formula join = new JoinFormula(binary, modifierFormula); + Formula merge = new MergeFormula(Mode.and, headFormula.child, join); + return new JoinFormula(headFormula.relation, merge); + } + + public static FeatureVector getBinaryBridgeFeatures(BinaryFormulaInfo bInfo) { + FeatureVector features = new FeatureVector(); + if (opts.useBinaryPredicateFeatures) + features.add("BridgeFn", "binary=" + bInfo.formula); + features.add("BridgeFn", "domain=" + bInfo.extractDomain(bInfo.formula)); + features.addWithBias("BridgeFn", "popularity", Math.log(bInfo.popularity + 1)); + return features; + } + + /** + * Lazy iterator for bridging - we always have the next derivation ready since it is possible that + * items in the list do not produce a derivation + */ + public class LazyBridgeFnDerivs extends MultipleDerivationStream { + + private ArrayList bridgingInfoList; + private int currIndex = 0; + + public LazyBridgeFnDerivs(ArrayList bridgingInfoList) { + this.bridgingInfoList = bridgingInfoList; + } + + @Override + public int estimatedSize() { + return bridgingInfoList.size() - currIndex; + } + + @Override + public Derivation createDerivation() { + if (currIndex == bridgingInfoList.size()) + return null; + Derivation res; + switch (description) { + case "unary": + res = nextUnary(); + break; + case "inject": + res = nextInject(); + break; + case "entity": + res = nextEntity(); + break; + default: + throw new RuntimeException("Bad description " + description); + } + if (opts.verbose >= 2) + LogInfo.logs("mode=%s,deriv=%s", description, res); + return res; + } + // not every BridgingInfo produces a derivation so we iterate until we find one + private Derivation nextEntity() { + + if (opts.verbose >= 3) + LogInfo.begin_track("Compute next entity"); + BridgingInfo currBridgingInfo = bridgingInfoList.get(currIndex++); + if (opts.verbose >= 2) + LogInfo.logs("BridgeFn.nextEntity: binary=%s, popularity=%s", currBridgingInfo.bInfo.formula, + currBridgingInfo.bInfo.popularity); + List> exampleInfo = generateExampleInfo(currBridgingInfo.ex, currBridgingInfo.c); // this is not done in text to text matcher so done here + Formula join = new JoinFormula(currBridgingInfo.bInfo.formula, currBridgingInfo.modifierDeriv.formula); + + Derivation res = new Derivation.Builder() + .withCallable(currBridgingInfo.c) + .formula(join) + .type(SemType.newAtomicSemType(currBridgingInfo.bInfo.expectedType1)) + .createDerivation(); + + if (SemanticFn.opts.trackLocalChoices) { + res.addLocalChoice( + String.format( + "BridgeFn: entity %s --> %s %s", + currBridgingInfo.bInfo.formula, + currBridgingInfo.modifierDeriv.startEndString(currBridgingInfo.ex.getTokens()), currBridgingInfo.modifierDeriv.formula)); + } + + if (opts.verbose >= 2) + LogInfo.logs("BridgeStringFn: %s", join); + + // features + res.addFeature("BridgeFn", "entity"); + res.addFeatures(getBinaryBridgeFeatures(currBridgingInfo.bInfo)); + + // Adds dependencies for every bridging relation/entity + if (FeatureExtractor.containsDomain("dependencyBridge")) { + addBridgeDependency(res, currBridgingInfo, "entity"); + } + + FeatureVector textMatchFeatures = textToTextMatcher.extractFeatures( + exampleInfo.get(0), exampleInfo.get(1), exampleInfo.get(2), + new HashSet<>(currBridgingInfo.bInfo.descriptions)); + res.addFeatures(textMatchFeatures); + + + if (opts.verbose >= 3) + LogInfo.end_track(); + return res; + } + + /* Adds dependencies for every bridging relation/entity. */ + private void addBridgeDependency(Derivation res, BridgingInfo currBridgingInfo, String type) { + List> deps = currBridgingInfo.ex.languageInfo.dependencyChildren; + Derivation entityDeriv = currBridgingInfo.modifierDeriv; + for (int currWord = 0; currWord < deps.size(); currWord++) { + if (entityDeriv.containsIndex(currWord)) { + continue; + } + for (DependencyEdge e : deps.get(currWord)) { + if (entityDeriv.containsIndex(e.modifier)) { + res.addFeature("dependencyBridge", "type=" + type + + "," + e.label + " -- " + + "relation=" + currBridgingInfo.bInfo.formula); + } + } + } + } + + private Derivation nextInject() { + BridgingInfo currBridgingInfo = bridgingInfoList.get(currIndex++); + if (opts.verbose >= 2) + LogInfo.logs("BridgingFn.nextInject: binary=%s, popularity=%s", currBridgingInfo.bInfo.formula, + currBridgingInfo.bInfo.popularity); + JoinFormula headFormula = (JoinFormula) Formulas.betaReduction(currBridgingInfo.headDeriv.formula); + + Formula bridgedFormula = buildBridgeFromCvt(headFormula, currBridgingInfo.modifierDeriv.formula, + currBridgingInfo.bInfo.formula); + Derivation res = new Derivation.Builder() + .withCallable(currBridgingInfo.c) + .formula(bridgedFormula) + .type(currBridgingInfo.headDeriv.type) + .createDerivation(); + if (SemanticFn.opts.trackLocalChoices) { + res.addLocalChoice( + String.format( + "BridgeFn: %s %s --> %s %s --> %s %s", + currBridgingInfo.headDeriv.startEndString(currBridgingInfo.ex.getTokens()), + currBridgingInfo.headDeriv.formula, + currBridgingInfo.ex.getTokens().subList(currBridgingInfo.c.child(0).end, currBridgingInfo.c.child(1).start), + currBridgingInfo.bInfo.formula, + currBridgingInfo.modifierDeriv.startEndString(currBridgingInfo.ex.getTokens()), currBridgingInfo.modifierDeriv.formula)); + } + + if (opts.verbose >= 3) + LogInfo.logs("BridgeFn: injecting %s to %s --> %s ", currBridgingInfo.modifierDeriv.formula, headFormula, bridgedFormula); + + String headModifierOrder = headFirst ? "head-modifier" : "modifier-head"; + res.addFeature("BridgeFn", + "inject_order=" + headModifierOrder + "," + "pos=" + + currBridgingInfo.ex.languageInfo.getCanonicalPos(currBridgingInfo.headDeriv.start) + "-" + + currBridgingInfo.ex.languageInfo.getCanonicalPos(currBridgingInfo.modifierDeriv.start) + ); + + + res.addFeature("BridgeFn", "binary=" + currBridgingInfo.bInfo.formula); + res.addFeature("BridgeFn", "domain=" + currBridgingInfo.bInfo.extractDomain(currBridgingInfo.bInfo.formula)); + res.addFeatureWithBias("BridgeFn", "popularity", Math.log(currBridgingInfo.bInfo.popularity + 1)); + /* Adds dependencies for every bridging relation/entity */ + if (FeatureExtractor.containsDomain("dependencyBridge")) { + addBridgeDependency(res, currBridgingInfo, "inject"); + } + return res; + } + + private Derivation nextUnary() { + BridgingInfo currBridgingInfo = bridgingInfoList.get(currIndex++); + + if (opts.verbose >= 2) + LogInfo.logs("BridgingFn.nextUnary: binary=%s, popularity=%s", currBridgingInfo.bInfo.formula, + currBridgingInfo.bInfo.popularity); + + Formula bridgedFormula = buildBridge(currBridgingInfo.headDeriv.formula, + currBridgingInfo.modifierDeriv.formula, currBridgingInfo.bInfo.formula); + + Derivation res = new Derivation.Builder() + .withCallable(currBridgingInfo.c) + .formula(bridgedFormula) + .type(currBridgingInfo.headDeriv.type) + .createDerivation(); + + if (SemanticFn.opts.trackLocalChoices) { + res.addLocalChoice( + String.format( + "BridgeFn: %s %s --> %s %s --> %s %s", + currBridgingInfo.headDeriv.startEndString(currBridgingInfo.ex.getTokens()), + currBridgingInfo.headDeriv.formula, + currBridgingInfo.ex.getTokens().subList(currBridgingInfo.c.child(0).end, currBridgingInfo.c.child(1).start), + currBridgingInfo.bInfo.formula, + currBridgingInfo.modifierDeriv.startEndString(currBridgingInfo.ex.getTokens()), currBridgingInfo.modifierDeriv.formula)); + } + + // Add features + res.addFeature("BridgeFn", "unary"); + res.addFeatures(getBinaryBridgeFeatures(currBridgingInfo.bInfo)); + + // head modifier POS tags + String headModifierOrder = headFirst ? "head-modifier" : "modifier-head"; + res.addFeature("BridgeFn", + "order=" + headModifierOrder + "," + + "pos=" + + currBridgingInfo.ex.languageInfo.getCanonicalPos(currBridgingInfo.headDeriv.start) + "-" + + currBridgingInfo.ex.languageInfo.getCanonicalPos(currBridgingInfo.modifierDeriv.start) + ); + + List> exampleInfo = generateExampleInfo(currBridgingInfo.ex, currBridgingInfo.c); // this is not done in text to text matcher so done here + + FeatureVector vector = textToTextMatcher.extractFeatures( + exampleInfo.get(0), exampleInfo.get(1), exampleInfo.get(2), + new HashSet<>(currBridgingInfo.bInfo.descriptions)); + + res.addFeatures(vector); + /* Adds dependencies for every bridging relation/entity */ + if (FeatureExtractor.containsDomain("dependencyBridge")) { + addBridgeDependency(res, currBridgingInfo, "unary"); + } + return res; + } + + @Override + public void remove() { + throw new RuntimeException("Does not support remove"); + } + } + + public class BridgingInfo implements Comparable { + public final Example ex; + public final Callable c; + public final BinaryFormulaInfo bInfo; + public final boolean headFirst; + public final Derivation headDeriv; + public final Derivation modifierDeriv; + + public BridgingInfo(Example ex, Callable c, BinaryFormulaInfo bInfo, boolean headFirst, Derivation headDeriv, Derivation modifierDeriv) { + this.ex = ex; + this.c = c; + this.bInfo = bInfo; + this.headFirst = headFirst; + this.headDeriv = headDeriv; + this.modifierDeriv = modifierDeriv; + } + + @Override + public int compareTo(BridgingInfo o) { + return fbFormulaInfo.compare(this.bInfo.formula, o.bInfo.formula); + } + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/BuildCanonicalIdMap.java b/src/edu/stanford/nlp/sempre/freebase/BuildCanonicalIdMap.java new file mode 100644 index 0000000..5c14f93 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/BuildCanonicalIdMap.java @@ -0,0 +1,146 @@ +package edu.stanford.nlp.sempre.freebase; + +import fig.basic.IOUtils; +import fig.basic.LogInfo; +import fig.basic.Option; +import fig.exec.Execution; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Motivation: the Freebase dump uses a mix of mids and ids. The mapping + * computed by this program will be used to standardize everything to a nice + * looking id or a mid. + *

+ * Input: raw RDF Freebase dump (ttl format) downloaded from the Freebase + * website. + *

+ * Output: tab-separated file containing mapping from mid/id to canonical id. + * fb:m.02mjmr fb:en.barack_obama fb:m.07t65 fb:en.united_nations + * fb:en.united_staff fb:en.united_nations ... To save space, only output mid/id + * which are mids or acceptable ids. + *

+ * Acceptable ids include: - Something that starts with fb:en. - Something that + * occurs in the arg1 position (for types and properties, excluding fb:m.* and + * fb:g.*). If there are multiple ids, just take the first one in the file. If + * there are no acceptable ids, then just take the mid. + * + * @author Percy Liang + */ +public class BuildCanonicalIdMap implements Runnable { + @Option public int maxInputLines = Integer.MAX_VALUE; + @Option(required = true) public String rawPath; + @Option(required = true) public String canonicalIdMapPath; + + Set allowableIds = new HashSet(); + + PrintWriter out; + int numMids = 0; + + void computeAllowableIds() { + // Compute allowable ids + LogInfo.begin_track("Compute allowable ids"); + try { + BufferedReader in = IOUtils.openIn(rawPath); + String line; + int numInputLines = 0; + while (numInputLines < maxInputLines && (line = in.readLine()) != null) { + numInputLines++; + if (numInputLines % 10000000 == 0) + LogInfo.logs("Read %s lines, %d allowable ids", numInputLines, allowableIds.size()); + String[] tokens = Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + if (!arg1.startsWith("fb:g.") && !arg1.startsWith("fb:m.")) + allowableIds.add(arg1); + } + LogInfo.logs("%d allowable ids", allowableIds.size()); + in.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.end_track(); + } + + void flush(String mid, List ids) { + if (ids.size() == 0) return; + numMids++; + + // Find the best id for this entity and put it first to use as the canonical id. + String bestId = mid; + for (String id : ids) { + if (id.startsWith("fb:en.") || allowableIds.contains(id)) { + bestId = id; + break; + } + } + + if (!bestId.equals(mid)) + out.println(mid + "\t" + bestId); + for (String id : ids) { + if (id.equals(bestId)) continue; + if (!(allowableIds.contains(id) || id.startsWith("fb:en."))) + continue; // To save space, only map ids which look reasonable + out.println(id + "\t" + bestId); + } + } + + public void run() { + computeAllowableIds(); + + // Map to ids + out = IOUtils.openOutHard(canonicalIdMapPath); + try { + BufferedReader in = IOUtils.openIn(rawPath); + String line; + int numInputLines = 0; + + // Current block of triples corresponds to a single mid. + String mid = null; + List ids = new ArrayList(); + + while (numInputLines < maxInputLines && (line = in.readLine()) != null) { + numInputLines++; + if (numInputLines % 10000000 == 0) + LogInfo.logs("Read %s lines, %d entities", numInputLines, numMids); + String[] tokens = Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + String property = tokens[1]; + String arg2 = tokens[2]; + + if (!arg1.startsWith("fb:m.")) continue; + if (!property.equals("fb:type.object.key")) continue; + + // Flush last block + if (!arg1.equals(mid)) { + flush(mid, ids); + + // Reset + mid = arg1; + ids.clear(); + } + + // Record information + arg2 = Utils.stringToRdf(arg2); + ids.add(arg2); + } + in.close(); + flush(mid, ids); + } catch (IOException e) { + throw new RuntimeException(e); + } + + out.close(); + } + + public static void main(String[] args) { + Execution.run(args, new BuildCanonicalIdMap()); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/BuildTypesMap.java b/src/edu/stanford/nlp/sempre/freebase/BuildTypesMap.java new file mode 100644 index 0000000..539551c --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/BuildTypesMap.java @@ -0,0 +1,64 @@ +package edu.stanford.nlp.sempre.freebase; + +import fig.basic.IOUtils; +import fig.basic.MapUtils; +import fig.basic.StrUtils; +import fig.basic.LogInfo; +import fig.basic.Option; +import fig.exec.Execution; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +/** + * Input: canonical Freebase dump + * fb:en.arnold_schwarzenegger fb:type.object.type fb:people.person. + * ... + * Output: map from ids to comma-separated list of types + * fb:en.arnold_schwarzenegger fb:people.person, fb:government.politician + * + * @author Percy Liang + */ +public class BuildTypesMap implements Runnable { + @Option public int maxInputLines = Integer.MAX_VALUE; + @Option(required = true, gloss = "Input") public String inPath; + @Option(required = true, gloss = "Output") public String outPath; + @Option(gloss = "keep only fb:en.*") public boolean keepOnlyEnIds = true; + + Map> types = new LinkedHashMap>(); + + public void run() { + LogInfo.begin_track("Reading %s", inPath); + try { + BufferedReader in = IOUtils.openIn(inPath); + String line; + int numLines = 0; + while ((line = in.readLine()) != null) { + String[] triple = Utils.parseTriple(line); + if (++numLines % 100000000 == 0) LogInfo.logs("%d lines", numLines); + if (triple == null) continue; + if (!triple[1].equals("fb:type.object.type")) continue; + if (keepOnlyEnIds && !triple[0].startsWith("fb:en.")) continue; + MapUtils.addToList(types, triple[0], triple[2]); + } + in.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.end_track(); + + LogInfo.begin_track("Writing to %s", outPath); + PrintWriter out = IOUtils.openOutHard(outPath); + for (Map.Entry> e : types.entrySet()) { + out.println(e.getKey() + "\t" + StrUtils.join(e.getValue(), ",")); + } + out.close(); + LogInfo.end_track(); + } + + public static void main(String[] args) { + Execution.run(args, new BuildTypesMap()); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/CanonicalizeExamples.java b/src/edu/stanford/nlp/sempre/freebase/CanonicalizeExamples.java new file mode 100644 index 0000000..ca55ca9 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/CanonicalizeExamples.java @@ -0,0 +1,83 @@ +package edu.stanford.nlp.sempre.freebase; + +import com.google.common.base.Function; +import edu.stanford.nlp.sempre.*; +import fig.basic.*; +import fig.exec.Execution; + +import java.io.PrintWriter; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Replaces all names (e.g., fb:m.02mjmr) with canonical identifiers (e.g., fb:en.barack_obama) + * For creating the dataset. + * + * @author Percy Liang + */ +public class CanonicalizeExamples implements Runnable { + @Option(required = true, gloss = "File mapping ids to canonical ids") + public String canonicalIdMapPath; + @Option(required = true, gloss = "Input path to examples to canonicalize (output to same directory) with .canonicalized extension") + public List examplePaths; + + Map canonicalIdMap; + + private String convert(String name) { + boolean reverse = false; + if (name.startsWith("!")) { + name = name.substring(1); + reverse = true; + } + name = MapUtils.get(canonicalIdMap, name, name); + if (reverse) name = "!" + name; + return name; + } + + public Formula convert(Formula formula) { + return formula.map( + new Function() { + public Formula apply(Formula formula) { + String name = Formulas.getNameId(formula); + if (name != null) { + name = convert(name); + return Formulas.newNameFormula(name); + } + return null; + } + }); + } + + public void run() { + canonicalIdMap = edu.stanford.nlp.sempre.freebase.Utils.readCanonicalIdMap(canonicalIdMapPath); + + for (String inPath : examplePaths) { + String outPath = inPath + ".canonicalized"; + LogInfo.logs("Converting %s => %s", inPath, outPath); + Iterator it = LispTree.proto.parseFromFile(inPath); + PrintWriter out = IOUtils.openOutHard(outPath); + while (it.hasNext()) { + LispTree tree = it.next(); + if (!"example".equals(tree.child(0).value)) + throw new RuntimeException("Bad: " + tree); + for (int i = 1; i < tree.children.size(); i++) { + LispTree subtree = tree.child(i); + if ("targetFormula".equals(subtree.child(0).value)) { + for (int j = 1; j < subtree.children.size(); j++) { + Formula formula = Formulas.fromLispTree(subtree.child(j)); + formula = convert(formula); + subtree.children.set(j, formula.toLispTree()); // Use converted formula + } + } + } + out.println(tree.toStringWrap(100)); + } + out.close(); + } + } + + public static void main(String[] args) { + Execution.run(args, new CanonicalizeExamples()); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/CanonicalizeIds.java b/src/edu/stanford/nlp/sempre/freebase/CanonicalizeIds.java new file mode 100644 index 0000000..7af1be1 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/CanonicalizeIds.java @@ -0,0 +1,80 @@ +package edu.stanford.nlp.sempre.freebase; + +import fig.basic.IOUtils; +import fig.basic.LogInfo; +import fig.basic.MapUtils; +import fig.basic.Option; +import fig.exec.Execution; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Map; + +/** + * Input: raw RDF Freebase dump (ttl format) downloaded from the Freebase + * website. Input: canonical id map file output by BuildCanonicalIdMap. + *

+ * Output: Freebase dump with all mids and ids standardized. + * + * @author Percy Liang + */ +public class CanonicalizeIds implements Runnable { + @Option public int maxInputLines = Integer.MAX_VALUE; + @Option(required = true) public String canonicalIdMapPath; + @Option(required = true, gloss = "Raw Freebase triples") + public String rawPath; + @Option(required = true, gloss = "Canonicalized Freebase triples") + public String canonicalizedPath; + + public void run() { + Map canonicalIdMap = Utils.readCanonicalIdMap(canonicalIdMapPath, maxInputLines); + + // Do conversion + LogInfo.begin_track("Convert"); + PrintWriter out = IOUtils.openOutHard(canonicalizedPath); + out.println(Utils.ttlPrefix); + try { + BufferedReader in = IOUtils.openIn(rawPath); + String line; + int numInputLines = 0; + while (numInputLines < maxInputLines && (line = in.readLine()) != null) { + numInputLines++; + if (numInputLines % 10000000 == 0) + LogInfo.logs("Read %s lines", numInputLines); + String[] tokens = Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + String property = tokens[1]; + String arg2 = tokens[2]; + + // Do some simple filtering + if (!property.startsWith("fb:")) continue; + if (property.contains("..")) + continue; // Freebase dumps started containing paths through CVTs, which we don't need + if (property.equals("fb:type.type.instance")) + continue; // Already have type.object.type, don't need reverse map explicitly + if (arg2.startsWith("\"") && !(arg2.endsWith("@en") || arg2.contains("^^xsd:"))) + continue; // Strings must be in English or xsd values (boolean, int, float, datetime) + + arg2 = Utils.quoteValues(arg2); // Fix numerical values + + // Convert everything to use canonical ids. + arg1 = MapUtils.get(canonicalIdMap, arg1, arg1); + property = MapUtils.get(canonicalIdMap, property, property); + arg2 = MapUtils.get(canonicalIdMap, arg2, arg2); + + Utils.writeTriple(out, arg1, property, arg2); + } + in.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + out.close(); + LogInfo.end_track(); + } + + public static void main(String[] args) { + Execution.run(args, new CanonicalizeIds()); + } +} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntityLexicon.java b/src/edu/stanford/nlp/sempre/freebase/EntityLexicon.java similarity index 63% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntityLexicon.java rename to src/edu/stanford/nlp/sempre/freebase/EntityLexicon.java index e3786fd..5918b25 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntityLexicon.java +++ b/src/edu/stanford/nlp/sempre/freebase/EntityLexicon.java @@ -1,19 +1,20 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; +package edu.stanford.nlp.sempre.freebase; -import edu.stanford.nlp.sempre.StringCache; -import edu.stanford.nlp.sempre.StringCacheUtils; -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.fbalignment.index.FbEntitySearcher; -import edu.stanford.nlp.sempre.fbalignment.index.FbIndexField; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.EntityLexicalEntry; -import edu.stanford.nlp.sempre.fbalignment.utils.FileUtils; -import edu.stanford.nlp.sempre.StringCache; -import edu.stanford.nlp.sempre.FreebaseSearch; -import edu.stanford.nlp.sempre.ValueFormula; -import edu.stanford.nlp.sempre.NameValue; +import edu.stanford.nlp.io.IOUtils; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.cache.StringCache; +import edu.stanford.nlp.sempre.cache.StringCacheUtils; +import edu.stanford.nlp.sempre.freebase.index.FbEntitySearcher; +import edu.stanford.nlp.sempre.freebase.index.FbIndexField; +import edu.stanford.nlp.sempre.freebase.lexicons.EntrySource; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.EntityLexicalEntry; +import edu.stanford.nlp.sempre.freebase.lexicons.TokenLevelMatchFeatures; +import edu.stanford.nlp.sempre.freebase.utils.FileUtils; import edu.stanford.nlp.stats.Counter; import edu.stanford.nlp.util.ArrayUtils; import edu.stanford.nlp.util.StringUtils; +import fig.basic.MapUtils; import fig.basic.Option; import org.apache.lucene.document.Document; import org.apache.lucene.queryparser.classic.ParseException; @@ -21,10 +22,12 @@ import org.apache.lucene.queryparser.classic.ParseException; import java.io.IOException; import java.util.*; -public class EntityLexicon { - public enum SearchStrategy { exact, inexact, fbsearch }; +public final class EntityLexicon { + public enum SearchStrategy { exact, inexact, fbsearch } public static class Options { + @Option(gloss = "Verbosity") + public int verbose = 0; @Option(gloss = "Number of results return by the lexicon") public int maxEntries = 1000; @Option(gloss = "Number of documents queried from Lucene") @@ -33,22 +36,34 @@ public class EntityLexicon { public String exactMatchIndex; @Option(gloss = "Path to the inexact match lucene index directory") public String inexactMatchIndex = "lib/lucene/4.4/inexact"; - @Option(gloss = "Cache path to the types path") - public String entityTypesPath = "localhost:4000:/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized.en-types"; @Option(gloss = "Cache path to the mid-to-id path") public String mid2idPath = "localhost:4000:/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map"; + @Option(gloss = "Path to entity popularity file") + public String entityPopularityPath; } public static Options opts = new Options(); + private static EntityLexicon entityLexicon; + public static EntityLexicon getInstance() { + if (entityLexicon == null) entityLexicon = new EntityLexicon(); + return entityLexicon; + } + FbEntitySearcher exactSearcher; // Lucene FbEntitySearcher inexactSearcher; // Lucene FreebaseSearch freebaseSearch; // Google's API StringCache mid2idCache; // Google's API spits back mids, which we need to convert to ids - StringCache entityTypesCache; // Given those ids, we retrieve the set of types + Map entityPopularityMap; + + private EntityLexicon() { + loadEntityPopularity(); + } public List lookupEntries(String query, SearchStrategy strategy) throws ParseException, IOException { + if (strategy == null) + throw new RuntimeException("No entity search strategy specified"); switch (strategy) { case exact: if (exactSearcher == null) exactSearcher = new FbEntitySearcher(opts.exactMatchIndex, opts.numOfDocs, "exact"); @@ -59,37 +74,54 @@ public class EntityLexicon { case fbsearch: if (freebaseSearch == null) freebaseSearch = new FreebaseSearch(); if (mid2idCache == null) mid2idCache = StringCacheUtils.create(opts.mid2idPath); - if (entityTypesCache == null) entityTypesCache = StringCacheUtils.create(opts.entityTypesPath); return lookupFreebaseSearchEntities(query); default: throw new RuntimeException("Unknown entity search strategy: " + strategy); } } + private void loadEntityPopularity() { + entityPopularityMap = new HashMap<>(); + if (opts.entityPopularityPath == null) return; + for (String line : IOUtils.readLines(opts.entityPopularityPath)) { + String[] tokens = line.split("\t"); + entityPopularityMap.put(tokens[0], Double.parseDouble(tokens[1])); + } + } + public List lookupFreebaseSearchEntities(String query) { FreebaseSearch.ServerResponse response = freebaseSearch.lookup(query); - List entities = new ArrayList(); + List entities = new ArrayList<>(); + if (response.error != null) { + throw new RuntimeException(response.error.toString()); + } + // num of words in query + int numOfQueryWords = query.split("\\s+").length; for (FreebaseSearch.Entry e : response.entries) { + if (entities.size() >= opts.maxEntries) break; // Note: e.id might not be the same one we're using (e.g., fb:en.john_f_kennedy_airport versus fb:en.john_f_kennedy_international_airport), // so get the one from our canonical mid2idCache String id = mid2idCache.get(e.mid); if (id == null) continue; // Skip if no ID (probably not worth referencing) - int distance = editDistance(query.toLowerCase(), e.name.toLowerCase()); // Is this actually useful? - Counter tokenEditDistanceFeatures = TokenLevelMatchFeatures.extractFeatures(query, e.name); - Set types = new HashSet(); - String typesStr = entityTypesCache.get(id); - if (typesStr != null) { - for (String type : typesStr.split(",")) - types.add(type); + // skip if it is a long phrase that is not an exact match + if (numOfQueryWords >= 4 && !query.toLowerCase().equals(e.name.toLowerCase())) { + continue; } - entities.add(new EntityLexicalEntry(query, query, Collections.singleton(e.name), new ValueFormula(new NameValue(id, e.name)), EntrySource.FBSEARCH, e.score, distance, types, tokenEditDistanceFeatures)); + + int distance = editDistance(query.toLowerCase(), e.name.toLowerCase()); // Is this actually useful? + Counter entityFeatures = TokenLevelMatchFeatures.extractFeatures(query, e.name); + double popularity = MapUtils.get(entityPopularityMap, id, 0d); + entityFeatures.incrementCount("text_popularity", Math.log(popularity + 1)); + entities.add(new EntityLexicalEntry(query, query, Collections.singleton(e.name), + new ValueFormula<>(new NameValue(id, e.name)), EntrySource.FBSEARCH, e.score, distance, + new FreebaseTypeLookup().getEntityTypes(id), entityFeatures)); } return entities; } public List lookupEntries(FbEntitySearcher searcher, String textDesc) throws ParseException, IOException { - List res = new LinkedList(); + List res = new ArrayList<>(); textDesc = textDesc.replaceAll("\\?", "\\\\?").toLowerCase(); List docs = searcher.searchDocs(textDesc); for (Document doc : docs) { @@ -98,11 +130,10 @@ public class EntityLexicon { String[] fbDescriptions = new String[]{doc.get(FbIndexField.TEXT.fieldName())}; String typesDesc = doc.get(FbIndexField.TYPES.fieldName()); - Set types = new HashSet(); + Set types = new HashSet<>(); if (typesDesc != null) { String[] tokens = typesDesc.split(","); - for (String token : tokens) - types.add(token); + Collections.addAll(types, tokens); } double popularity = Double.parseDouble(doc.get(FbIndexField.POPULARITY.fieldName())); @@ -110,7 +141,6 @@ public class EntityLexicon { Counter tokenEditDistanceFeatures = TokenLevelMatchFeatures.extractFeatures(textDesc, fbDescriptions[0]); if ((popularity > 0 || distance == 0) && TokenLevelMatchFeatures.diffSetSize(textDesc, fbDescriptions[0]) < 4) { - res.add(new EntityLexicalEntry(textDesc, textDesc, ArrayUtils.asSet(fbDescriptions), formula, EntrySource.LUCENE, popularity, distance, types, tokenEditDistanceFeatures)); } } @@ -124,18 +154,17 @@ public class EntityLexicon { String[] nameTokens = FileUtils.omitPunct(name).split("\\s+"); StringBuilder querySb = new StringBuilder(); - for (int i = 0; i < queryTokens.length; ++i) - querySb.append(queryTokens[i] + " "); + for (String queryToken : queryTokens) + querySb.append(queryToken).append(" "); StringBuilder nameSb = new StringBuilder(); - for (int i = 0; i < nameTokens.length; ++i) - nameSb.append(nameTokens[i] + " "); + for (String nameToken : nameTokens) + nameSb.append(nameToken).append(" "); return StringUtils.editDistance(querySb.toString().trim(), nameSb.toString().trim()); } - - public static class LexicalEntryComparator implements Comparator { + public static class LexicalEntryComparator implements Comparator { @Override public int compare(LexicalEntry arg0, LexicalEntry arg1) { @@ -148,13 +177,8 @@ public class EntityLexicon { if (arg0.distance > arg1.distance) return 1; return 0; - - } - - } - } diff --git a/src/edu/stanford/nlp/sempre/freebase/ExecuteExamples.java b/src/edu/stanford/nlp/sempre/freebase/ExecuteExamples.java new file mode 100644 index 0000000..2c1692b --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/ExecuteExamples.java @@ -0,0 +1,84 @@ +package edu.stanford.nlp.sempre.freebase; + +import edu.stanford.nlp.sempre.*; + +import com.google.common.base.Function; +import fig.basic.LispTree; +import fig.basic.LogInfo; +import fig.basic.Option; +import fig.exec.Execution; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Executes all examples and reports any badness. + * Was used to prepare the Free917 dataset. + * + * @author Percy Liang + */ +public class ExecuteExamples implements Runnable { + @Option(required = true, gloss = "Input path to examples to canonicalize") + public List examplesPaths; + + SparqlExecutor executor; + Map cache = new HashMap(); + + private boolean queryReturnsResults(Formula formula) { + // If counting, look inside to make sure the actual set is non-empty. + if (formula instanceof AggregateFormula) + formula = ((AggregateFormula) formula).child; + + Executor.Response response = cache.get(formula); + if (response == null) + cache.put(formula, response = executor.execute(formula, null)); + if (!(response.value instanceof ListValue) || + ((ListValue) response.value).values.size() == 0) { + LogInfo.logs("BAD QUERY: %s => %s", formula, response.value); + return false; + } else { + LogInfo.logs("GOOD QUERY: %s => %s", formula, response.value); + return true; + } + } + + // Test each individual nested NameFormula. + public Formula test(Formula formula) { + return formula.map( + new Function() { + public Formula apply(Formula formula) { + String name = Formulas.getNameId(formula); + if (name != null) { + if (name.startsWith("!")) name = name.substring(1); + queryReturnsResults(Formulas.newNameFormula(name)); + } + return null; + } + }); + } + + public void run() { + executor = new SparqlExecutor(); + for (String path : examplesPaths) { + Iterator it = LispTree.proto.parseFromFile(path); + while (it.hasNext()) { + LispTree tree = it.next(); + if (!"example".equals(tree.child(0).value)) + throw new RuntimeException("Bad: " + tree); + for (int i = 1; i < tree.children.size(); i++) { + if ("targetFormula".equals(tree.child(i).child(0).value)) { + Formula formula = Formulas.fromLispTree(tree.child(i).child(1)); + formula = test(formula); + queryReturnsResults(formula); + } + } + } + } + } + + public static void main(String[] args) { + Execution.run(args, new ExecuteExamples(), "SparqlExecutor", SparqlExecutor.opts); + } +} diff --git a/src/edu/stanford/nlp/sempre/FbFormulasInfo.java b/src/edu/stanford/nlp/sempre/freebase/FbFormulasInfo.java similarity index 60% rename from src/edu/stanford/nlp/sempre/FbFormulasInfo.java rename to src/edu/stanford/nlp/sempre/freebase/FbFormulasInfo.java index b92b6a4..ab81feb 100644 --- a/src/edu/stanford/nlp/sempre/FbFormulasInfo.java +++ b/src/edu/stanford/nlp/sempre/freebase/FbFormulasInfo.java @@ -1,6 +1,9 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.freebase; import com.google.common.base.Joiner; +import edu.stanford.nlp.stats.ClassicCounter; +import edu.stanford.nlp.stats.Counter; +import edu.stanford.nlp.sempre.*; import fig.basic.LispTree; import fig.basic.LogInfo; import fig.basic.MapUtils; @@ -17,48 +20,38 @@ import java.util.*; * * @author jonathanberant */ -public class FbFormulasInfo { +public final class FbFormulasInfo { // Everyone should use the singleton. private static FbFormulasInfo fbFormulaInfo; + public static FbFormulasInfo getSingleton() { - if (fbFormulaInfo == null) fbFormulaInfo = new FbFormulasInfo(); + if (fbFormulaInfo == null) { + fbFormulaInfo = new FbFormulasInfo(); + } return fbFormulaInfo; } - private static Set binaryFormulasTouched = new HashSet(); //all binary formulas touched during parsing in join and bridge - - private FreebaseInfo freebaseInfo=null; - public Map binaryFormulaInfoMap = new HashMap(); - public Map unaryFormulaInfoMap = new HashMap(); - private Map> typeToNumericalPredicates = new HashMap>(); - private Map> atomicExtype2ToBinaryMap = new HashMap>(); //contains map to all atomic properties - private Map> extype2ToNonCvtBinaryMap = new HashMap>(); //contains map to all binaries for which extype 1 is not a CVT - private Map> cvtExpansionsMap = new HashMap>(); - private Map> cvtTypeToBinaries = new HashMap>(); + private FreebaseInfo freebaseInfo = null; + public Map binaryFormulaInfoMap = new HashMap<>(); + public Map unaryFormulaInfoMap = new HashMap<>(); + private Map> typeToNumericalPredicates = new HashMap<>(); + private Map> atomicExtype2ToBinaryMap = new HashMap<>(); // contains map to all atomic properties + private Map> extype2ToNonCvtBinaryMap = new HashMap<>(); // contains map to all binaries for which extype 1 is not a CVT + private Map> cvtExpansionsMap = new HashMap<>(); + private Map> cvtTypeToBinaries = new HashMap<>(); + private Comparator formulaComparator; private FbFormulasInfo() { try { freebaseInfo = FreebaseInfo.getSingleton(); - loadFormulaInfo(); + loadFormulaInfo(); computeNumericalPredicatesMap(); - } catch (IOException e) { + } catch (IOException | NumberFormatException e) { throw new RuntimeException(e); - } catch (NumberFormatException e) { - throw new RuntimeException(e); - } + } } - public static void touchBinaryFormula(Formula binary) { - binaryFormulasTouched.add(binary); - } - - public static int numOfTouchedBinaries() { - return binaryFormulasTouched.size(); - } - - - /** * Map that given a type provides all Freebase predicates that have that type * as expected type 2 and a number for expected type 1 @@ -76,16 +69,16 @@ public class FbFormulasInfo { } private void computeReverseFormulaInfo() { - List entriesToAdd = new LinkedList(); + List entriesToAdd = new LinkedList<>(); for (Formula formula : binaryFormulaInfoMap.keySet()) { BinaryFormulaInfo info = binaryFormulaInfoMap.get(formula); - Formula reverseFormula = reverseFormula(formula); + Formula reverseFormula = Formulas.reverseFormula(formula); if (!binaryFormulaInfoMap.containsKey(reverseFormula)) { entriesToAdd.add( - new BinaryFormulaInfo( - reverseFormula, - info.expectedType2, info.expectedType1, info.unitId, info.unitDesc, info.descriptions, info.popularity)); + new BinaryFormulaInfo( + reverseFormula, + info.expectedType2, info.expectedType1, info.unitId, info.unitDesc, info.descriptions, info.popularity)); } } LogInfo.log("Adding reverse formulas: " + entriesToAdd.size()); @@ -102,7 +95,7 @@ public class FbFormulasInfo { return unaryFormulaInfoMap.get(formula); } - private void loadFormulaInfo() throws NumberFormatException, IOException { + private void loadFormulaInfo() throws IOException { LogInfo.begin_track("Loading formula info..."); LogInfo.log("Adding schema properties"); @@ -115,42 +108,58 @@ public class FbFormulasInfo { computeReverseFormulaInfo(); LogInfo.log("Current number of binary formulas: " + binaryFormulaInfoMap.size()); for (BinaryFormulaInfo info : binaryFormulaInfoMap.values()) { - MapUtils.addToSet(atomicExtype2ToBinaryMap, info.expectedType2, info.formula); - if(!isCvt(info.expectedType1)) { + + MapUtils.addToList(atomicExtype2ToBinaryMap, info.expectedType2, info.formula); + if (!isCvt(info.expectedType1)) { addMappingFromType2ToFormula(info.expectedType2, info.formula); } } LogInfo.log("Generate formulas through CVTs"); - generateCvtFormulas(); //generate formulas for CVTs + generateCvtFormulas(); // generate formulas for CVTs LogInfo.log("Current number of binary formulas: " + binaryFormulaInfoMap.size()); + // we first sort by popularity + Comparator comparator = getPopularityComparator(); + sortType2ToBinaryMaps(comparator); LogInfo.end_track(); } + public void sortType2ToBinaryMaps(Comparator comparator) { + this.formulaComparator = comparator; + for (List binaries: atomicExtype2ToBinaryMap.values()) + Collections.sort(binaries, comparator); + + for (List binaries: extype2ToNonCvtBinaryMap.values()) + Collections.sort(binaries, comparator); + + } + + public int compare(Formula f1, Formula f2) { + return formulaComparator.compare(f1, f2); + } + /** * Adding mapping from type 2 to formula - makes sure to insert just one of the 2 equivalent formulas if they exist - * @param type2 - * @param formula */ private void addMappingFromType2ToFormula(String type2, Formula formula) { - MapUtils.addToSet(extype2ToNonCvtBinaryMap, type2, formula); + MapUtils.addToList(extype2ToNonCvtBinaryMap, type2, formula); } private void generateCvtFormulas() throws FileNotFoundException { - List toAdd = new ArrayList(); + List toAdd = new ArrayList<>(); for (BinaryFormulaInfo innerInfo : binaryFormulaInfoMap.values()) { - if (isCvt(innerInfo.expectedType1)) { //if expected type 1 is a CVT + if (isCvt(innerInfo.expectedType1)) { // if expected type 1 is a CVT MapUtils.addToSet(cvtTypeToBinaries, innerInfo.expectedType1, innerInfo.formula); - Set outers = atomicExtype2ToBinaryMap.get(innerInfo.expectedType1); //find those whose expected type 2 is that CVT + List outers = atomicExtype2ToBinaryMap.get(innerInfo.expectedType1); // find those whose expected type 2 is that CVT for (Formula outer : outers) { BinaryFormulaInfo outerInfo = binaryFormulaInfoMap.get(outer); if (!isLegalCvt(innerInfo.formula, outer)) continue; - //build new formula + // build new formula LambdaFormula cvtFormula = new LambdaFormula("x", new JoinFormula(outer, new JoinFormula(innerInfo.formula, new VariableFormula("x")))); BinaryFormulaInfo newFormulaInfo = binaryFormulaInfoMap.get(cvtFormula); @@ -159,7 +168,7 @@ public class FbFormulasInfo { if (exType1 == null) throw new RuntimeException("Missing expected type 1 for formula: " + outer); - List newDescriptions = new LinkedList(); + List newDescriptions = new LinkedList<>(); newDescriptions.add(outerInfo.descriptions.get(0)); newDescriptions.add(innerInfo.descriptions.get(0)); @@ -182,90 +191,32 @@ public class FbFormulasInfo { return false; if (!FreebaseInfo.isReverseProperty(inner.toString()) && FreebaseInfo.isReverseProperty(outer.toString())) return false; - //if (isEquivalent(reverseFormula(outer), inner)) //don't expand if reverse is equivalent - //return false; return true; } - /** Reverse the order of arguments */ - //Takes in a |rawFormula| which is a function. - public static Formula reverseFormula(Formula rawFormula) { - if (rawFormula instanceof ValueFormula) { - @SuppressWarnings({"unchecked"}) - ValueFormula vf = (ValueFormula) rawFormula; - return reverseNameFormula(vf); - } else if (rawFormula instanceof LambdaFormula) { - // Convert (lambda x (relation1 (relation2 (var x)))) <=> (lambda x (!relation2 (!relation1 (var x)))) - // Note: currently only handles chains. Make this more generic. - LambdaFormula formula = (LambdaFormula) rawFormula; - if (formula.body instanceof JoinFormula || formula.body instanceof VariableFormula) - return new LambdaFormula(formula.var, reverseChain(formula.body, new VariableFormula(formula.var))); - else - return new ReverseFormula(formula); - } else { - return new ReverseFormula(rawFormula); - //throw new RuntimeException("Not handled: " + rawFormula); - } - } - - // Helper function for reverseFormula(). - private static Formula reverseChain(Formula source, Formula result) { - if (source instanceof JoinFormula) { - JoinFormula join = (JoinFormula) source; - return reverseChain(join.child, new JoinFormula(reverseFormula(join.relation), result)); - } else if (source instanceof VariableFormula) { - return result; - } else { - throw new RuntimeException("Not handled: " + source); - } - } - - // !fb:people.person.place_of_birth <=> fb:people.person.place_of_birth - private static ValueFormula reverseNameFormula(ValueFormula formula) { - String id = formula.value.id; - return new ValueFormula( - new NameValue(FreebaseInfo.isReverseProperty(id) ? - id.substring(1) : "!" + id)); - } - /** supports chains only */ - public boolean hasOpposite(String formula) { - LispTree tree = LispTree.proto.parseFromString(formula); + public boolean hasOpposite(String formula) { return hasOpposite(LispTree.proto.parseFromString(formula)); } + public boolean hasOpposite(Formula formula) { return hasOpposite(formula.toLispTree()); } + private boolean hasOpposite(LispTree tree) { if (tree.isLeaf()) { String fbProperty = FreebaseInfo.isReverseProperty(tree.value) ? tree.value.substring(1) : tree.value; - return freebaseInfo.fbPropertyHasOpposite(fbProperty); + return freebaseInfo.propertyHasOpposite(fbProperty); } else { // Un-reverse everything. String binary1 = tree.child(2).child(0).value; binary1 = FreebaseInfo.isReverseProperty(binary1) ? binary1.substring(1) : binary1; String binary2 = tree.child(2).child(1).child(0).value; binary2 = FreebaseInfo.isReverseProperty(binary2) ? binary2.substring(1) : binary2; - return freebaseInfo.fbPropertyHasOpposite(binary1) && freebaseInfo.fbPropertyHasOpposite(binary2); - } - } - - /** supports chains only */ - public boolean hasOpposite(Formula formula) { - LispTree tree = formula.toLispTree(); - if (tree.isLeaf()) { - String fbProperty = FreebaseInfo.isReverseProperty(tree.value) ? tree.value.substring(1) : tree.value; - return freebaseInfo.fbPropertyHasOpposite(fbProperty); - } else { - // Un-reverse everything. - String binary1 = tree.child(2).child(0).value; - binary1 = FreebaseInfo.isReverseProperty(binary1) ? binary1.substring(1) : binary1; - String binary2 = tree.child(2).child(1).child(0).value; - binary2 = FreebaseInfo.isReverseProperty(binary2) ? binary2.substring(1) : binary2; - return freebaseInfo.fbPropertyHasOpposite(binary1) && freebaseInfo.fbPropertyHasOpposite(binary2); + return freebaseInfo.propertyHasOpposite(binary1) && freebaseInfo.propertyHasOpposite(binary2); } } /** supports chains only */ public boolean isReversed(Formula formula) { LispTree tree = formula.toLispTree(); - if (tree.isLeaf()) + if (tree.isLeaf()) return FreebaseInfo.isReverseProperty(tree.value); - else + else return FreebaseInfo.isReverseProperty(tree.child(2).child(0).value); } @@ -280,6 +231,9 @@ public class FbFormulasInfo { return equivalentFormula(tree); } + // two formulas can be equivalent because there are two names for every edge using the reverse label + //fb:people.person.place_of_birth --> !fb:location.location.people_born_here + //!fb:people.person.place_of_birth --> fb:location.location.people_born_here public Formula equivalentFormula(LispTree tree) { if (tree.isLeaf()) { @@ -299,6 +253,9 @@ public class FbFormulasInfo { } } + //input: |binary1|=fb:people.place_lived.location, + // |binary2|=fb:people.person.places_lived, |reverse|=true + //output: (lambda x (!fb:people.place_lived.location (!fb:people.person.places_lived (var x)))) public static Formula buildLambdaFormula(String binary1, String binary2, boolean reverse) { Formula binary1Formula = reverse ? Formulas.newNameFormula("!" + binary1) : Formulas.newNameFormula(binary1); @@ -308,33 +265,28 @@ public class FbFormulasInfo { return new LambdaFormula("x", join2); } - public boolean isEquivalent(Formula formula1, Formula formula2) { - if (formula1.equals(formula2)) - return true; - if (hasOpposite(formula1)) { - Formula equiv1 = equivalentFormula(formula1); - return equiv1.equals(formula2); - } - return false; - } + + //for binary formulas that are paths in the graph, if formula1 is a path s-->t + //then formula2 is the opposite if it is the path t-->s + // fb:people.person.place_of_birth is the opposite of !fb:people.person.place_of_birth + // fb:people.person.place_of_birth is the opposite of fb: private boolean isOpposite(Formula formula1, Formula formula2) { - if(isReversed(formula1) && !isReversed(formula2)) { + if (isReversed(formula1) && !isReversed(formula2)) { String formula1Desc = formula1.toString().substring(1); return formula1Desc.equals(formula2.toString()); } - if(isReversed(formula2) && !isReversed(formula1)) { + if (isReversed(formula2) && !isReversed(formula1)) { String formula2Desc = formula2.toString().substring(1); return formula2Desc.equals(formula1.toString()); } - if(hasOpposite(formula1)) { + if (hasOpposite(formula1)) { Formula equivalentFormula = equivalentFormula(formula1); - if(isReversed(equivalentFormula)) { + if (isReversed(equivalentFormula)) { String equivalentFormaulDesc = equivalentFormula.toString().substring(1); return equivalentFormaulDesc.equals(formula2.toString()); - } - else { + } else { String formula2Desc = formula2.toString().substring(1); return formula2Desc.equals(equivalentFormula.toString()); } @@ -342,20 +294,12 @@ public class FbFormulasInfo { return false; } - public Set getIncludedTypesInclusive(String subtype) { - return freebaseInfo.getIncludedTypesInclusive(subtype); + public List getBinariesForType2(String type) { + return MapUtils.get(extype2ToNonCvtBinaryMap, type, new ArrayList()); } - public Set getSubtypesExclusive(String supertype) { - return freebaseInfo.getSubTypesExclusive(supertype); - } - - public Set getBinariesForType2(String type) { - return MapUtils.get(extype2ToNonCvtBinaryMap, type, new HashSet()); - } - - public Set getAtomicBinariesForType2(String type) { - return MapUtils.get(atomicExtype2ToBinaryMap, type, new HashSet()); + public List getAtomicBinariesForType2(String type) { + return MapUtils.get(atomicExtype2ToBinaryMap, type, new ArrayList()); } public boolean isCvtFormula(BinaryFormulaInfo info) { @@ -370,39 +314,93 @@ public class FbFormulasInfo { return MapUtils.getSet(cvtTypeToBinaries, cvt); } + //For a binary lambda formula that goes through CVTs, find all binaries that can + //be injected to this lambda binary formula. + //example: + //input: (lambda (fb:people.person.places_lived (fb:people.place_lived.location (var x)))) + //output: fb:people.place_lived/start_date, fb:people.place_lived.end_date public List getInjectableBinaries(Formula formula) { - List res = new ArrayList(); - if(!(formula instanceof LambdaFormula)) return res; + List res = new ArrayList<>(); + if (!(formula instanceof LambdaFormula)) return res; LambdaFormula lambdaFormula = (LambdaFormula) formula; Formula first = ((JoinFormula) lambdaFormula.body).relation; Formula second = ((JoinFormula) ((JoinFormula) lambdaFormula.body).child).relation; Set binaryFormulas = expandCvts(getBinaryInfo(first).expectedType2); - for(Formula binaryFormula: binaryFormulas) { - if(!second.equals(binaryFormula) && - !isOpposite(first, binaryFormula)) { + for (Formula binaryFormula : binaryFormulas) { + if (!second.equals(binaryFormula) && !isOpposite(first, binaryFormula)) { res.add(binaryFormula); } } return res; } - public Map getFormulaInfoMap() { - return binaryFormulaInfoMap; - } - public boolean isCvt(String type) { return freebaseInfo.isCvt(type); } + public Comparator getPopularityComparator() { + Counter counter = new ClassicCounter<>(); + for (Formula binaryFormula : binaryFormulaInfoMap.keySet()) + counter.incrementCount(binaryFormula, binaryFormulaInfoMap.get(binaryFormula).popularity); + + return new FormulaByCounterComparator(counter); + } + + public class FormulaByCounterComparator implements Comparator { + + private Counter fCounter; + + public FormulaByCounterComparator(Counter fCounter) { + this.fCounter = fCounter; + } + public int compare(Formula f1, Formula f2) { + double count1 = fCounter.getCount(f1); + double count2 = fCounter.getCount(f2); + if (count1 > count2) return -1; + if (count1 < count2) return +1; + double pop1 = binaryFormulaInfoMap.get(f1).popularity; + double pop2 = binaryFormulaInfoMap.get(f2).popularity; + if (pop1 > pop2) return -1; + if (pop1 < pop2) return +1; + return 0; + } + public double getCount(Formula f) { return fCounter.getCount(f); } + } + + public class FormulaByFeaturesComparator implements Comparator { + + private Params params; + + public FormulaByFeaturesComparator(Params params) { + this.params = params; + } + public int compare(Formula f1, Formula f2) { + + FeatureVector features1 = BridgeFn.getBinaryBridgeFeatures(fbFormulaInfo.getBinaryInfo(f1)); + FeatureVector features2 = BridgeFn.getBinaryBridgeFeatures(fbFormulaInfo.getBinaryInfo(f2)); + + double score1 = features1.dotProduct(params); + double score2 = features2.dotProduct(params); + if (score1 > score2) return -1; + if (score1 < score2) return +1; + double pop1 = binaryFormulaInfoMap.get(f1).popularity; + double pop2 = binaryFormulaInfoMap.get(f2).popularity; + if (pop1 > pop2) return -1; + if (pop1 < pop2) return +1; + return 0; + } + } + + //Information from freebase about binary formulas public static class BinaryFormulaInfo { - public Formula formula; - public String expectedType1; - public String expectedType2; - public String unitId = ""; - public String unitDesc = ""; - public List descriptions=new LinkedList(); - public double popularity; + public Formula formula; //fb:people.person.place_of_birth + public String expectedType1; //fb:people.person + public String expectedType2; //fb:location.location + public String unitId = ""; //fb:en.meter + public String unitDesc = ""; //Meter + public List descriptions = new LinkedList<>(); // "place of birth" + public double popularity; //Number of instances of binary in KB: 16184.0 public BinaryFormulaInfo(Formula formula, String exType1, String exType2, List descs, double popularity) { this.formula = formula; @@ -424,15 +422,15 @@ public class FbFormulasInfo { } public String toString() { return formula.toString() + "\t" + popularity + "\t" + expectedType1 + "\t" + expectedType2 + "\t" + unitId + "\t" - + unitDesc + "\t" + Joiner.on("###").join(descriptions); + + unitDesc + "\t" + Joiner.on("###").join(descriptions); } public String toReverseString() { - return reverseFormula(formula).toString() + "\t" + popularity + "\t" + expectedType2 + "\t" + expectedType1 + "\t" + unitId + "\t" - + unitDesc + "\t" + Joiner.on("###").join(descriptions); + return Formulas.reverseFormula(formula).toString() + "\t" + popularity + "\t" + expectedType2 + "\t" + expectedType1 + "\t" + unitId + "\t" + + unitDesc + "\t" + Joiner.on("###").join(descriptions); } public static List tokenizeFbDescription(String fbDesc) { - List res = new ArrayList(); + List res = new ArrayList<>(); String[] tokens = fbDesc.split("\\s+"); for (String token : tokens) { token = token.replace("(", ""); @@ -445,15 +443,23 @@ public class FbFormulasInfo { public boolean isComplete() { if (formula == null || expectedType1 == null || expectedType2 == null || - expectedType1.equals("") || expectedType2.equals("") || descriptions == null || - descriptions.size()==0 || - popularity == 0.0) + expectedType1.equals("") || expectedType2.equals("") || descriptions == null || + descriptions.size() == 0 || + popularity == 0.0) return false; return true; } public SemType getSemType() { - return new FuncSemType(new EntitySemType(expectedType2), new EntitySemType(expectedType1)); + return SemType.newFuncSemType(expectedType2, expectedType1); + } + + public String extractDomain(Formula binary) { + LispTree tree = binary.toLispTree(); + String property = tree.isLeaf() ? tree.value : tree.child(2).child(0).value; + if (property.startsWith("!")) + property = property.substring(1); + return property.substring(0, property.indexOf('.')); } } @@ -465,7 +471,7 @@ public class FbFormulasInfo { public Set types; public UnaryFormulaInfo(Formula formula, double popularity, - List descriptions, Set types) { + List descriptions, Set types) { this.formula = formula; this.popularity = popularity; @@ -474,8 +480,8 @@ public class FbFormulasInfo { } public boolean isComplete() { - if (formula == null || descriptions == null || descriptions.size()==0 || - popularity == 0.0) + if (formula == null || descriptions == null || descriptions.size() == 0 || + popularity == 0.0) return false; return true; } @@ -485,8 +491,7 @@ public class FbFormulasInfo { } public String getRepresentativeDescrption() { - if(descriptions.get(0).contains("/") && - descriptions.size()>1) + if (descriptions.get(0).contains("/") && descriptions.size() > 1) return descriptions.get(1); return descriptions.get(0); } diff --git a/src/edu/stanford/nlp/sempre/freebase/FilterFreebase.java b/src/edu/stanford/nlp/sempre/freebase/FilterFreebase.java new file mode 100644 index 0000000..fa71c0e --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/FilterFreebase.java @@ -0,0 +1,218 @@ +package edu.stanford.nlp.sempre.freebase; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import edu.stanford.nlp.sempre.Formula; +import edu.stanford.nlp.sempre.Formulas; +import fig.basic.*; +import fig.exec.Execution; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +/** + * Input: canonicalized Freebase ttl file. Input: example files. Output: subset + * of the ttl file that only involves the referenced properties. + * + * @author Percy Liang + */ +public class FilterFreebase implements Runnable { + @Option(required = true, gloss = "Canonicalized Freebase dump") + public String inPath; + @Option public int maxInputLines = Integer.MAX_VALUE; + @Option(gloss = "Examples files (keep properties that show up in these files)") + public List examplesPaths = new ArrayList(); + + @Option(gloss = "Keep only type entries involving these") + public List keepTypesPaths = new ArrayList(); + @Option(gloss = "Keep these properties") + public List keepPropertiesPaths = new ArrayList(); + @Option(gloss = "Ignore these properties") + public List notKeepPropertiesPaths = new ArrayList(); + + @Option(gloss = "Schema properties to keep") + public HashSet schemaProperties = new HashSet( + ImmutableList.of( + "fb:type.property.schema", + "fb:type.property.unit", + "fb:type.property.expected_type", + "fb:type.property.reverse_property", + "fb:freebase.type_hints.mediator", + "fb:freebase.type_hints.included_types" + )); + + @Option(gloss = "General properties that we should keep") + public HashSet generalProperties = new HashSet( + ImmutableList.of( + "fb:type.object.type", + "fb:type.object.name", + "fb:measurement_unit.dated_integer.number", + "fb:measurement_unit.dated_integer.year" + )); + + // Set this if we want to make a small Freebase. + @Option(gloss = "If true, keep general properties only for entities seen with the other keepProperties (uses much more memory, but results in smaller output)") + public boolean keepGeneralPropertiesOnlyForSeenEntities = false; + + @Option public boolean keepAllProperties = false; + + // Keep only type assertions involving these types. + // If empty, don't filter. + Set keepTypes = new LinkedHashSet(); + + // These are the properties for which we should keep all entity pairs. Derived from many sources. + // Should never be empty. + Set keepProperties = new LinkedHashSet(); + + // Entities that we saw (only needed if we need to use them to filter general properties later). + Set seenEntities = new HashSet(); + + // Fill out |keepProperties| + private void readKeep() { + LogInfo.begin_track("readKeep"); + + // Always keep schema + keepProperties.addAll(schemaProperties); + + // General properties to keep + if (!keepGeneralPropertiesOnlyForSeenEntities) + keepProperties.addAll(generalProperties); + + // Keep properties mentioned in examples + for (String path : examplesPaths) { + LogInfo.logs("Reading %s", path); + Iterator it = LispTree.proto.parseFromFile(path); + while (it.hasNext()) { + LispTree tree = it.next(); + if (!"example".equals(tree.child(0).value)) + throw new RuntimeException("Bad: " + tree); + for (int i = 1; i < tree.children.size(); i++) { + if ("targetFormula".equals(tree.child(i).child(0).value)) { + Formula formula = Formulas.fromLispTree(tree.child(i).child(1)); + keepProperties.addAll(Formulas.extractAtomicFreebaseElements(formula)); + } + } + } + } + + // Keep types + for (String path : keepTypesPaths) + for (String type : IOUtils.readLinesHard(path)) + keepTypes.add(type); + + // Keep and not keep properties + for (String path : keepPropertiesPaths) + for (String property : IOUtils.readLinesHard(path)) + keepProperties.add(property); + for (String path : notKeepPropertiesPaths) + for (String property : IOUtils.readLinesHard(path)) + keepProperties.remove(property); + + PrintWriter out = IOUtils.openOutHard(Execution.getFile("keepProperties")); + for (String property : keepProperties) + out.println(property); + out.close(); + LogInfo.logs("Keeping %s properties", keepProperties.size()); + LogInfo.end_track(); + } + + private void filterTuples() { + LogInfo.begin_track("filterTuples"); + TDoubleMap propertyCounts = new TDoubleMap(); + + PrintWriter out = IOUtils.openOutHard(Execution.getFile("0.ttl")); + out.println(Utils.ttlPrefix); + + try { + BufferedReader in = IOUtils.openIn(inPath); + String line; + int numInputLines = 0; + int numOutputLines = 0; + while (numInputLines < maxInputLines && (line = in.readLine()) != null) { + numInputLines++; + if (numInputLines % 10000000 == 0) + LogInfo.logs("filterTuples: Read %s lines, written %d lines", numInputLines, numOutputLines); + String[] tokens = Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + String property = tokens[1]; + String arg2 = tokens[2]; + if (!keepAllProperties && !keepProperties.contains(property)) continue; + + if (keepGeneralPropertiesOnlyForSeenEntities) { + seenEntities.add(arg1); + seenEntities.add(arg2); + } + + // Additional filtering of characters that Virtuoso can't index (we would need to be escape these). + if (Utils.isUrl(arg2)) continue; + if (Utils.identifierContainsStrangeCharacters(arg1) || Utils.identifierContainsStrangeCharacters(arg2)) + continue; + + Utils.writeTriple(out, arg1, property, arg2); + + propertyCounts.incr(property, 1); + numOutputLines++; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + // Make a second pass to only output general properties. + if (keepGeneralPropertiesOnlyForSeenEntities) { + LogInfo.begin_track("Second pass to output general properties for the %d seen entities", seenEntities.size()); + try { + BufferedReader in = IOUtils.openIn(inPath); + String line; + int numInputLines = 0; + int numOutputLines = 0; + while (numInputLines < maxInputLines && (line = in.readLine()) != null) { + numInputLines++; + if (numInputLines % 10000000 == 0) + LogInfo.logs("filterTuples: Read %s lines, written %d lines", numInputLines, numOutputLines); + String[] tokens = Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + String property = tokens[1]; + String arg2 = tokens[2]; + if (!generalProperties.contains(property)) continue; + if (!seenEntities.contains(arg1)) continue; + + // Only keep types that matter + if (keepTypes.size() != 0 && property.equals("fb:type.object.type") && !keepTypes.contains(arg2)) continue; + + Utils.writeTriple(out, arg1, property, arg2); + + numOutputLines++; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + LogInfo.end_track(); + } + + out.close(); + + // Output property statistics + PrintWriter propertyCountsOut = IOUtils.openOutHard(Execution.getFile("propertyCounts")); + List.Entry> entries = Lists.newArrayList(propertyCounts.entrySet()); + Collections.sort(entries, propertyCounts.entryValueComparator()); + for (TDoubleMap.Entry e : entries) { + propertyCountsOut.println(e.getKey() + "\t" + e.getValue()); + } + propertyCountsOut.close(); + + LogInfo.end_track(); + } + + public void run() { + readKeep(); + filterTuples(); + } + + public static void main(String[] args) { + Execution.run(args, new FilterFreebase()); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/Free917Converter.java b/src/edu/stanford/nlp/sempre/freebase/Free917Converter.java new file mode 100644 index 0000000..f47a914 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/Free917Converter.java @@ -0,0 +1,727 @@ +package edu.stanford.nlp.sempre.freebase; + +import edu.stanford.nlp.io.IOUtils; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.BinaryFormulaInfo; +import edu.stanford.nlp.sempre.MergeFormula.Mode; +import edu.stanford.nlp.sempre.freebase.utils.FileUtils; +import edu.stanford.nlp.sempre.freebase.utils.FormatConverter; +import edu.stanford.nlp.stats.ClassicCounter; +import edu.stanford.nlp.stats.Counter; +import edu.stanford.nlp.util.StringUtils; +import fig.basic.LispTree; +import fig.basic.LogInfo; +import fig.basic.MapUtils; +import fig.basic.Option; +import fig.exec.Execution; +import fig.prob.SampleUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +/** + * One-time hack that converts the Cai & Yates dataset to our format. + * @author Jonathan Berant + */ +public class Free917Converter implements Runnable { + + private Counter argnumCounter = new ClassicCounter(); + private FbFormulasInfo formulaInfo = FbFormulasInfo.getSingleton(); + private Set cvts; + @Option(gloss = "Input path to examples to canonicalize") + public String inDir; + @Option(gloss = "Input path to examples to canonicalize") + public String outDir; + @Option(gloss = "Input path to examples to canonicalize") + public String entityInfoFile; + @Option(gloss = "Input path to examples to canonicalize") + public String cvtFile; + @Option(gloss = "Input path to examples to canonicalize") + public String midToIdFile; + + @Override + public void run() { + try { + String inQuestionsFile = inDir + "question-and-logical-form-917/dataset-all-917.txt"; + String inNpFile = inDir + "fixed-np-manually.txt"; + String outQuestionFile = outDir + "dataset-all-917_corrected.txt"; + String outNpFile = outDir + "fixed-np-manually_corrected.txt"; + String outputPrefix = outDir + "free917"; + String free917EntityInfoFile = outDir + "entityInfo.txt"; + String free917MissingEntitiesFile = outDir + "missingEntities.txt"; + cvts = FileUtils.loadSet(cvtFile); + + correctErrors(inQuestionsFile, inNpFile, outQuestionFile, outNpFile); + convertExampleFile(outQuestionFile, outputPrefix); + genreateEntityInfoFile(outNpFile, entityInfoFile, free917EntityInfoFile, free917MissingEntitiesFile); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + public static void main(String[] args) throws IOException { + Execution.run(args, new Free917Converter()); + } + + private void correctErrors(String inQuestionsFile, String inNpFile, + String outQuestionFile, String outNpFile) throws IOException { + + // manual corrections of question file + int i = 0; + PrintWriter questionWriter = IOUtils.getPrintWriter(outQuestionFile); + for (String line : IOUtils.readLines(inQuestionsFile)) { + if (line.equals("(lambda $0 /type/int (exists $1 (/award/ranking@rank@year@note@list@item:t $0 /type/datetime/2000:/type/datetime $1 /en/fortune_500:/award/ranked_list /en/monsanto:/award/ranked_item)))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (/award/ranking@rank@year@note@list@item:t $0 /type/datetime/2000:/type/datetime $1 $2 /en/fortune_500:/award/ranked_list /en/monsanto:/award/ranked_item)))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (exists $4 (/medicine/disease/survival_rates&/medicine/survival_rate@gender@race@years@rate@disease_stage:t /en/prostate_cancer:/medicine/disease $1 $2 $0 $3 $4))))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (exists $4 (/medicine/disease/survival_rates&/medicine/survival_rate@gender@race@years@rate@disease_stage:t /en/prostate_cancer:/medicine/disease $-1 $1 $2 $3 $0 $4))))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/soccer/football_league_participation@team@league@from@to:t $1 $0 $2 $3)))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/soccer/football_league_participation@team@league@from@to:t /en/real_madrid:/soccer/football_team $1 $0 $2 $3)))))"); + else if (line.equals("(lambda $0 /common/topic (/tv/tv_program@languages:t /base/ranker/rankerurlname/firefly$002f143400:/tv/tv_program $0))")) + questionWriter.println("(lambda $0 /common/topic (/tv/tv_program@languages:t /m/014v3t:/tv/tv_program $0))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/film/film/estimated_budget&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/edward_scissorhands:/film/film $1 $0 $2))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/film/film/estimated_budget&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/edward_scissorhands:/film/film $-1 $1 $0 $2))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/film/film/estimated_budget&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/transformers:/film/film $1 $0 $2))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/film/film/estimated_budget&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/transformers:/film/film $-1 $1 $0 $2))))"); + else if (line.equals("(lambda $0 /location/location (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/library/public_library/address&/location/mailing_address@street_address@street_address_2@citytown@postal_code@state_province_region@country:t /m/02ncllz:/library/public_library $1 $2 $0 $3 $4 $5)))))))")) + questionWriter.println("(lambda $0 /location/location (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/library/public_library/address&/location/mailing_address@street_address@street_address_2@citytown@postal_code@state_province_region@country:t /m/02ncllz:/library/public_library $-1 $1 $2 $0 $3 $4 $5)))))))"); + else if (line.equals("who won ali–frazier ii")) + questionWriter.println("who won muhammad ali vs. joe frazier ii"); + else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (/base/boxing/match_boxer_relationship@match@boxer@winner_won@points:t $1 $0 $2 $3)))))")) { + if (i++ == 0) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (/boxing/match_boxer_relationship@match@boxer@winner_won@points:t /en/ali-frazier_ii:/boxing/boxing_match $1 $0 /type/boolean/true $2 $3)))))"); + else + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (/boxing/match_boxer_relationship@match@boxer@winner_won@points:t /m/0kvlz:/boxing/boxing_match $1 $0 $2 $3)))))"); + } else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (exists $6 (/base/boxing/boxing_title_tenure@champion@weight@from@to@defenses@title@notes:t $0 $1 $2 $3 $4 $5 $6))))))))")) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (exists $6 (/boxing/boxing_title_tenure@champion@weight@from@to@defenses@title@notes:t $0 $1 $2 $3 $4 /m/0chgh2j:/boxing/boxing_title $5 $6))))))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/business_operation/revenue&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/motorola:/business/business_operation $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/business_operation/revenue&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/motorola:/business/business_operation $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $1 /common/topic (exists $2 (exists $3 (/business/business_operation/net_profit&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/procter_gamble:/business/business_operation $1 $2 $3))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/business/business_operation/net_profit&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/procter_gamble:/business/business_operation $-1 $1 $0 $2))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (/business/business_operation/revenue&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/viacom:/business/business_operation /un/2009:/type/datetime $0 $1)))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (/business/business_operation/revenue&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/viacom:/business/business_operation /type/datetime/2009:/type/datetime $1 $0)))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/business/business_operation/operating_income&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/j_c_penney:/business/business_operation $1 $0 $2))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/business/business_operation/operating_income&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/j_c_penney:/business/business_operation $-1 $1 $0 $2))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/belgium:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/belgium:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/iowa:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/iowa:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/location/statistical_region/major_exports&/location/imports_exports_by_industry@amount@currency@date@industry:t /en/madagascar:/location/statistical_region $1 $2 $3 $0)))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/location/statistical_region/major_exports&/location/imports_exports_by_industry@amount@currency@date@industry:t /en/madagascar:/location/statistical_region $-1 $1 $2 $3 $0)))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/africa:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/africa:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/asia:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/asia:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/earth:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/earth:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/europe:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/europe:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/earth:/location/statistical_region $0 $1 $2))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (/location/statistical_region/population&/measurement_unit/dated_integer@number@year@source:t /en/earth:/location/statistical_region $-1 $0 $1 $2))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/tv/regular_tv_appearance@actor@character@series@from@to@special_performance_type@seasons:t /en/ron_glass:/tv/tv_actor $1 $0 $2 $3 $4 $5)))))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/tv/regular_tv_appearance@actor@character@series@from@to@special_performance_type@seasons:t /en/ron_glass:/tv/tv_actor $1 $2 $0 $3 $4 $5)))))))"); + else if (line.equals("(lambda $0 /common/topic (/fashion/garment@specialization_of:t /fashion/garment:/fashion/garment $0))")) + questionWriter.println("(lambda $0 /common/topic (/fashion/garment@specialization_of:t /en/knickerbockers:/fashion/garment $0))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (exists $4 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $2 /en/golden_globe_award_for_best_motion_picture_-_drama:/award/award_category $3 $0 $4))))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (exists $4 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $2 /en/golden_globe_award_for_best_motion_picture_-_drama:/award/award_category $3 $4 $0))))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/celebrities/sexual_orientation_phase@celebrity@start@end@sexual_orientation:t /en/britney_spears:/celebrities/celebrity $1 $2 $0))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/celebrities/sexual_orientation_phase@celebrity@start@end@sexual_orientation:t /en/britney_spears:/celebrities/celebrity $1 $2 $3 $0))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/projects/project_participation@project@participant@role@from_date@to_date:t /m/0gk9x46:/projects/project /en/francesco_sabatini:/projects/project_participant $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/projects/project_participation@project@participant@role@from_date@to_date:t /m/0gk9x46:/projects/project $1 /en/francesco_sabatini:/projects/project_participant $2 $3 $0))))"); + else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (exists $6 (/government/government_position_held@office_holder@office_position_or_title@governmental_body@district_represented@appointed_by@from@to@jurisdiction_of_office@legislative_sessions:t $0 /en/united_states_senator:/government/government_office_or_title $1 $2 $3 $4 $5 /en/colorado:/government/governmental_jurisdiction $6))))))))")) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (exists $6 (/government/government_position_held@office_holder@office_position_or_title@governmental_body@district_represented@appointed_by@from@to@jurisdiction_of_office@legislative_sessions:t $0 /en/united_states_senator:/government/government_office_or_title $1 $2 /en/colorado:/government/governmental_jurisdiction $3 $4 $5 $6 $7 $8))))))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (/ice_hockey/hockey_previous_roster_position@team@player@from@to:t /en/montreal_canadiens:/ice_hockey/hockey_team /en/christopher_higgins:/ice_hockey/hockey_player $0 $1)))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (/ice_hockey/hockey_previous_roster_position@team@player@from@to:t /en/montreal_canadiens:/ice_hockey/hockey_team /en/christopher_higgins:/ice_hockey/hockey_player $1 $2 $0)))"); + else if (line.equals("(lambda $0 /location/location (/organization/organization@headquarters:t /en/apple_inc:/organization/organization $0))")) + questionWriter.println("(lambda $0 /location/location (exists $1 (exists $2 (exists $3 (exists $4 (/organization/organization/headquarters&/location/mailing_address@citytown@race@years@rate@disease_stage:t /en/apple_inc:/organization/organization $-1 $0))))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (/celebrities/substance_abuse_problem@substance@start@end@celebrity:t /en/cocaine:/celebrities/abused_substance $0 $1 /en/robin_williams:/celebrities/celebrity)))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (/celebrities/substance_abuse_problem@substance@start@end@celebrity:t /en/cocaine:/celebrities/abused_substance $1 $2 $0 /en/robin_williams:/celebrities/celebrity)))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/travel/transportation@travel_destination@mode_of_transportation@transport_operator@transport_terminus:t /en/paris:/travel/travel_destination $1 $0 $2))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/travel/transportation@travel_destination@mode_of_transportation@transport_operator@transport_terminus:t /en/paris:/travel/travel_destination $1 $2 $0))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (/martial_arts/martial_arts_certification@person@qualification@certifying_body@date@art:t /en/cathy_landers:/martial_arts/martial_artist /en/fifth_degree:/martial_arts/martial_arts_qualification $1 $0 /en/seishindo_kenpo:/martial_arts/martial_art)))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (/martial_arts/martial_arts_certification@person@qualification@certifying_body@date@art:t /en/cathy_landers:/martial_arts/martial_artist /en/fifth_degree:/martial_arts/martial_arts_qualification $1 $2 $0 /en/seishindo_kenpo:/martial_arts/martial_art)))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (exists $3 (/american_football/football_historical_roster_position@player@team@from@to@number@position_s:t /en/david_akers:/american_football/football_player /en/philadelphia_eagles:/american_football/football_team $1 $2 $0 $3)))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (exists $3 (/american_football/football_historical_roster_position@player@team@from@to@number@position_s:t /en/david_akers:/american_football/football_player /en/philadelphia_eagles:/american_football/football_team $1 $2 $3 $0 $4)))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (exists $4 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $2 /m/04d215m:/award/award_category $3 $0 $4))))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (exists $4 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $2 /m/04d215m:/award/award_category $3 $4 $0))))))"); + else if (line.equals("(count $0 (/architecture/architect@structures_designed:t /en/frank_lloyd_wright:/architecture/architect $0))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (/architecture/architect@structures_designed:t /en/frank_lloyd_wright:/architecture/architect $0)))"); + else if (line.equals("(lambda $0 /common/topic (/conferences/conference_subject@series_of_conferences_about_this:t /en/mathematics:/conferences/conference_subject $0))")) + questionWriter.println("(count $0 (/conferences/conference_subject@series_of_conferences_about_this:t /en/mathematics:/conferences/conference_subject $0))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/ritz_cracker:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/ritz_cracker:/business/brand $1 $2 $0))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (exists $2 (exists $3 (/american_football/player_game_statistics@player@season@team@games@starts@as_of_week:t /en/donovan_mcnabb:/american_football/football_player /en/2008_nfl_season:/sports/sports_league_season $1 $0 $2 $3)))))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (exists $2 (exists $3 (/american_football/player_game_statistics@player@season@team@games@starts@as_of_week:t /en/donovan_mcnabb:/american_football/football_player /en/2008_nfl_season:/sports/sports_league_season $1 $2 $0 $3)))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (/royalty/chivalric_order_position_tenure@order@chivalric_office@from@until@officer:t /en/order_of_the_most_holy_annunciation:/royalty/order_of_chivalry /en/grand_master:/royalty/chivalric_office $1 $2 $0))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (/royalty/chivalric_order_position_tenure@order@chivalric_office@from@until@officer:t /en/order_of_the_most_holy_annunciation:/royalty/order_of_chivalry /en/grand_master:/royalty/chivalric_office $1 $2 $3 $0))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (exists $3 (/film/performance@actor@film@special_performance_type@character@character_note:t $1 /en/charlies_angels:/film/film $2 $0 $3)))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (exists $3 (/film/performance@actor@film@special_performance_type@character@character_note:t $1 /en/charlies_angels:/film/film $2 $3 $0 $4)))))"); + else if (line.equals("(lambda $0 /location/location (/library/public_library@address:t /en/mitchell_public_library:/library/public_library $0))")) + questionWriter.println("(lambda $0 /location/location (/library/public_library/address&/location/mailing_address@citytown@street_address:t /m/0j9by57:/library/public_library $-1 $1 $0))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/chips_ahoy:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/chips_ahoy:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (/media_common/dedication@dedicated_by@dedicated_to@work_dedicated@notes:t /en/joseph_haydn:/media_common/dedicator $1 $0 $2))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (/media_common/dedication@dedicated_by@dedicated_to@work_dedicated@notes:t /en/wolfgang_amadeus_mozart:/media_common/dedicator $1 /en/joseph_haydn:/media_common/dedicator $2 $0 $3))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/government/political_party_tenure@politician@from@to@party:t /en/grover_cleveland:/government/politician $1 $2 $0))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/government/political_party_tenure@politician@from@to@party:t /en/grover_cleveland:/government/politician $1 $2 $3 $0))))"); + else if (line.equals("(lambda $0 /location/location (/organization/organization@headquarters:t /en/h_r_block:/organization/organization $0))")) + questionWriter.println("(lambda $0 /location/location (/organization/organization/headquarters&/location/mailing_address@citytown@street_address:t /en/h_r_block:/organization/organization $-1 $0))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/country_time:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/country_time:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/girl_scouts_of_the_usa:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (/organization/organization@date_founded:t /en/girl_scouts_of_the_usa:/organization/organization $0))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/tostitos:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/tostitos:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/girl_scouts_of_the_usa:/business/brand $0 $1 $2))))")) + questionWriter.println("(lambda $0 /common/topic (/organization/organization@founders:t /en/girl_scouts_of_the_usa:/business/brand $0))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /m/02r3cjp:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /m/02r3cjp:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/stove_top_stuffing:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/stove_top_stuffing:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/oreo:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/oreo:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (exists $3 (/cvg/musical_game_song_relationship@download@game@platforms@song@release_date:t $1 /en/guitar_hero_aerosmith:/cvg/musical_game $2 $0 $3)))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (exists $3 (/cvg/musical_game_song_relationship@download@game@platforms@song@release_date:t $1 /en/guitar_hero_aerosmith:/cvg/musical_game $2 $3 $0 $4)))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (/olympics/olympic_athlete_affiliation@athlete@country@olympics@sport:t /m/04dnjr9:/olympics/olympic_athlete $1 /en/1992_summer_olympics:/olympics/olympic_games $0)))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (/olympics/olympic_athlete_affiliation@athlete@country@olympics@sport:t /m/04dnjr9:/olympics/olympic_athlete $1 $2 /en/1992_summer_olympics:/olympics/olympic_games $3 $0)))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/tv/regular_tv_appearance@actor@character@series@from@to@special_performance_type@seasons:t /en/jerry_seinfeld:/tv/tv_actor $1 $0 $2 $3 $4 $5)))))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/tv/regular_tv_appearance@actor@character@series@from@to@special_performance_type@seasons:t /en/jerry_seinfeld:/tv/tv_actor $1 $2 $0 $3 $4 $5 $6)))))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/basketball/basketball_roster_position@number@player@position@team:t $1 /en/richard_hamilton:/basketball/basketball_player $2 $0))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/basketball/basketball_roster_position@number@player@position@team:t $1 /en/richard_hamilton:/basketball/basketball_player $2 $3 $0))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/crystal_light:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/crystal_light:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/kool-aid:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/kool-aid:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 /type/datetime/1981:/type/datetime $2 /en/danny_devito:/award/award_winner $0 $3)))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 /type/datetime/1981:/type/datetime $2 $3 /en/danny_devito:/award/award_winner $4 $0 $5)))))"); + else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (/american_football/football_historical_roster_position@player@team@from@to@number@position_s:t $0 /en/green_bay_packers:/american_football/football_team $1 $2 $3 /en/quarterback:/american_football/football_position)))))")) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (/american_football/football_historical_roster_position@player@team@from@to@number@position_s:t $0 /en/green_bay_packers:/american_football/football_team $1 $2 $3 $4 /en/quarterback:/american_football/football_position)))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (/event/speech_or_presentation@event@speech_topic@speaker_s@type_or_format_of_presentation@presented_work@date:t $1 /en/world_war_ii:/event/speech_topic $2 $3 $0 $4))))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (/event/speech_or_presentation@event@speech_topic@speaker_s@type_or_format_of_presentation@presented_work@date:t $1 /en/world_war_ii:/event/speech_topic $2 $3 $4 $0 $5))))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (/celebrities/substance_abuse_problem@substance@start@end@celebrity:t /en/cocaine:/celebrities/abused_substance $1 $2 $0))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (/celebrities/substance_abuse_problem@substance@start@end@celebrity:t /en/cocaine:/celebrities/abused_substance $1 $2 $3 $0))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/doritos:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/doritos:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (/event/speech_or_presentation@event@speech_topic@speaker_s@type_or_format_of_presentation@presented_work@date:t $1 $2 /en/winston_churchill:/event/public_speaker $3 $0 $4))))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (exists $3 (exists $4 (/event/speech_or_presentation@event@speech_topic@speaker_s@type_or_format_of_presentation@presented_work@date:t $1 $2 /en/winston_churchill:/event/public_speaker $3 $4 $0 $5))))))"); + else if (line.equals("(lambda $0 /common/topic (/transportation/bridge@bridge_type:t /en/suspension_bridge:/transportation/bridge $0))")) + questionWriter.println("(lambda $0 /common/topic (/transportation/bridge@bridge_type:t /en/manhattan_bridge:/transportation/bridge $0))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/martial_arts/martial_arts_certification@person@qualification@certifying_body@date@art:t /en/christopher_adams:/martial_arts/martial_artist /en/black_belt:/martial_arts/martial_arts_qualification $1 $2 $0))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/martial_arts/martial_arts_certification@person@qualification@certifying_body@date@art:t /en/christopher_adams:/martial_arts/martial_artist $1 /en/black_belt:/martial_arts/martial_arts_qualification $2 $3 $4 $0))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/capri_sun:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/capri_sun:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (/organization/leadership@person@title@as_of_date@organization@role@from@to:t $0 /en/chief_executive_officer:/type/text $1 /en/save-a-lot:/organization/organization $2 $3 $4))))))")) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (/organization/leadership@person@title@as_of_date@organization@role@from@to:t $0 $1 $2 /en/save-a-lot:/organization/organization $3 /en/chief_executive_officer:/type/text $4 $5))))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/barbie:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/barbie:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/capn_crunch:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/capn_crunch:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/gatorade:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/gatorade:/business/brand $1 $2 $0 $3))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/mountain_dew:/business/brand $1 $0 $2))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (/business/company_brand_relationship@brand@company@from_date@to_date:t /en/mountain_dew:/business/brand $1 $2 $0 $3))))"); + // entity issues + else if (line.equals("(lambda $0 /common/topic (/music/track@producer:t /m/0l16j8:/music/track $0))")) + questionWriter.println("(lambda $0 /common/topic (/music/recording@producer:t /m/0l16j8:/music/recording $0))"); + else if (line.equals("(lambda $0 /people/person (/book/written_work@author:t /m/03crp32:/book/written_work $0))")) + questionWriter.println("(lambda $0 /people/person (/book/written_work@author:t /m/067y_k7:/book/written_work $0))"); + else if (line.equals("(lambda $0 /common/topic (/book/literary_genre@books_in_this_genre:t $0 /en/the_hound_of_the_baskervilles:/book/book))")) + questionWriter.println("(lambda $0 /common/topic (/media_common/literary_genre@books_in_this_genre:t $0 /en/the_hound_of_the_baskervilles:/book/book))"); + else if (line.equals("(count $0 (/book/literary_genre@books_in_this_genre:t /en/computer_programming:/book/literary_genre $0))")) + questionWriter.println("(count $0 (/media_common/literary_genre@books_in_this_genre:t /en/computer_programming:/book/literary_genre $0))"); + else if (line.equals("(lambda $0 /type/int (/freebase/type_profile@instance_count:t /film/film_actor:/freebase/type_profile $0))")) + questionWriter.println("(lambda $0 /type/int (/freebase/type_profile@instance_count:t /film/actor:/freebase/type_profile $0))"); + else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (/olympics/olympic_medal_honor@country@event@medal@medalist@olympics:t $1 /view/en/tennis_at_the_1896_summer_olympics_mens_singles:/olympics/olympic_event_competition /en/gold_medal:/olympics/olympic_medal $0 $2))))")) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (/olympics/olympic_medal_honor@country@event@medal@medalist@olympics:t $1 /en/tennis_at_the_1896_summer_olympics_mens_singles:/olympics/olympic_event_competition /en/gold_medal:/olympics/olympic_medal $0 $2))))"); + else if (line.equals("(lambda $0 /type/datetime (/amusement_parks/ride@opened:t /m/03mfjrv:/amusement_parks/ride $0))")) + questionWriter.println("(lambda $0 /type/datetime (/amusement_parks/ride@opened:t /m/0flmt0:/amusement_parks/ride $0))"); + else if (line.equals("(lambda $0 /common/topic (/base/dinosaur/dinosaur@diet:t /en/ceratopsia/-/base/dinosaur:/base/dinosaur/dinosaur $0))")) + questionWriter.println("(lambda $0 /common/topic (/base/dinosaur/dinosaur@diet:t /en/ceratopsia:/base/dinosaur/dinosaur $0))"); + else if (line.equals("(lambda $0 /location/location (/base/dinosaur/dinosaur_location@dinosaur_s:t $0 /en/barosaurus/-/base/dinosaur:/base/dinosaur/dinosaur))")) + questionWriter.println("(lambda $0 /location/location (/base/dinosaur/dinosaur_location@dinosaur_s:t $0 /en/barosaurus:/base/dinosaur/dinosaur))"); + else if (line.equals("(lambda $0 /common/topic (/chemistry/chemical_element@symbol:t /authority/us/gov/hhs/fda/srs-unii/fxs1by2pgl:/chemistry/chemical_element $0))")) + questionWriter.println("(lambda $0 /common/topic (/chemistry/chemical_element@symbol:t /m/025sw5g:/chemistry/chemical_element $0))"); + else if (line.equals("(count $0 (/amusement_parks/park@annual_visits:t /en/magic_kingdom:/amusement_parks/park $0))")) + questionWriter.println("(lambda $0 /type/int (/amusement_parks/park@annual_visits:t /en/magic_kingdom:/amusement_parks/park $0))"); + else if (line.equals("(lambda $0 /common/topic (/organization/organization@slogan:t /business/cik/0001011006:/organization/organization $0))")) + questionWriter.println("(lambda $0 /common/topic (/organization/organization@slogan:t /m/019rl6:/organization/organization $0))"); + else if (line.equals("(lambda $0 /location/location (/location/location@containedby:t /base/usnris/item/86000083:/location/location $0))")) + questionWriter.println("(lambda $0 /location/location (/location/location@containedby:t /m/019zhn:/location/location $0))"); + else if (line.equals("(lambda $0 /common/topic (/opera/opera@language:t /base/imslp/65847:/opera/opera $0))")) + questionWriter.println("(lambda $0 /common/topic (/opera/opera@language:t /m/09hvx:/opera/opera $0))"); + else if (line.equals("(lambda $0 /people/person (/film/film@costume_design_by:t /source/allocine/fr/film/132663:/film/film $0))")) + questionWriter.println("(lambda $0 /people/person (/film/film@costume_design_by:t /m/04jpg2p:/film/film $0))"); + else if (line.equals("(lambda $0 /people/person (/architecture/architectural_style@architects:t /en/bauhaus:/architecture/architectural_style $0))")) + questionWriter.println("(lambda $0 /people/person (/architecture/architectural_style@architects:t /en/international_style:/architecture/architectural_style $0))"); + else if (line.equals("(lambda $0 /common/topic (/astronomy/star@temperature_k:t /en/polaris:/astronomy/star $0))")) + questionWriter.println("(lambda $0 /common/topic (/astronomy/star@temperature_k:t /m/0kjyrc7:/astronomy/star $0))"); + else if (line.equals("(lambda $0 /common/topic (/book/literary_series@fictional_universe:t /en/the_lord_of_the_rings:/book/literary_series $0))")) + questionWriter.println("(lambda $0 /common/topic (/fictional_universe/work_of_fiction@setting:t /en/the_lord_of_the_rings:/book/literary_series $0))"); + else if (line.equals("(lambda $0 /common/topic (/chemistry/chemical_element@melting_point:t /quotationsbook/subject/gold:/chemistry/chemical_element $0))")) + questionWriter.println("(lambda $0 /common/topic (/chemistry/chemical_element@melting_point:t /m/025rs2z:/chemistry/chemical_element $0))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (exists $3 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $0 /en/guardian_first_book_award:/award/award_category $2 /en/everything_is_illuminated:/award/award_winning_work $3)))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (exists $3 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $0 /en/guardian_first_book_award:/award/award_category $2 $3 /en/everything_is_illuminated:/award/award_winning_work $4)))))"); + else if (line.equals("(lambda $0 /type/datetime (exists $1 (exists $2 (exists $3 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $0 /en/hugo_award_for_best_novel:/award/award_category $2 /en/harry_potter_and_the_goblet_of_fire:/award/award_winning_work $3)))))")) + questionWriter.println("(lambda $0 /type/datetime (exists $1 (exists $2 (exists $3 (/award/award_honor@notes_description@year@award@award_winner@honored_for@ceremony:t $1 $0 /en/hugo_award_for_best_novel:/award/award_category $2 $3 /en/harry_potter_and_the_goblet_of_fire:/award/award_winning_work $3)))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (/american_football/football_roster_position@team@player@position@number:t /en/baltimore_ravens:/american_football/football_team /en/ray_lewis:/american_football/football_player $0 $1)))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (/american_football/football_historical_roster_position@team@player@position_s@number:t /en/baltimore_ravens:/american_football/football_team /en/ray_lewis:/american_football/football_player $0 $1)))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/award/award_nomination@award@year@award_nominee@nominated_for@notes_description:t /en/peoples_choice_award_for_favorite_comedy_movie:/award/award_category $1 $2 $0 $3)))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/award/award_nomination@award@year@award_nominee@nominated_for@notes_description:t /en/peoples_choice_award_for_favorite_comedy_movie:/award/award_category $1 $2 $3 $0 $4)))))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (/award/ranking@rank@year@note@list@item:t $0 /type/datetime/2010:/type/datetime $1 /en/fortune_500:/award/ranked_list /en/target_corporation:/award/ranked_item)))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (/award/ranking@rank@year@note@list@item:t $0 /type/datetime/2010:/type/datetime $1 $2 /en/fortune_500:/award/ranked_list /en/target_corporation:/award/ranked_item)))"); + else if (line.equals("(lambda $0 /type/int (exists $1 (/baseball/baseball_roster_position@position@team@player@number:t $1 /en/boston_red_sox:/baseball/baseball_team /en/kevin_youkilis:/baseball/baseball_player $0)))")) + questionWriter.println("(lambda $0 /type/int (exists $1 (/sports/sports_team_roster@position@team@player@number:t $1 /en/boston_red_sox:/baseball/baseball_team /en/kevin_youkilis:/baseball/baseball_player $0)))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/basketball/basketball_roster_position@number@player@position@team:t $1 /en/keyon_dooling:/basketball/basketball_player $0 $2))))")) + questionWriter.println("(lambda $0 /common/topic (/basketball/basketball_player@position_s:t /en/keyon_dooling:/basketball/basketball_player $0))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (/business/sponsorship@sponsored_by@from@to@sponsored_recipient:t $0 $1 $2 /en/gatorade:/business/sponsored_recipient))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/business/sponsorship@sponsored_by@from@to@sponsored_recipient:t /en/gatorade:/business/sponsored_recipient $1 $2 $3 $0))))"); + else if (line.equals("(lambda $0 /common/topic (/freebase/domain_profile@category:t /film:/freebase/domain_profile $0))")) + questionWriter.println("(lambda $0 /common/topic (/freebase/domain_profile@category:t /m/010s:/freebase/domain_profile $0))"); + else if (line.equals("(count $0 (exists $1 (exists $2 (/soccer/football_league_participation@team@league@from@to:t $0 /en/uefa:/soccer/football_league $1 $2))))")) + questionWriter.println("(count $0 (exists $1 (exists $2 (/sports/sports_league_participation@team@league@from@to:t $0 /en/uefa:/soccer/football_league $1 $2))))"); + else if (line.equals("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/tv/tv_regular_personal_appearance@to@from@appearance_type@person@program@seasons:t $1 $2 /en/newscaster:/tv/non_character_role $0 /en/abc_news:/tv/tv_program $3)))))")) + questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (exists $3 (/business/employment_tenure@person@company@title:t $0 /en/abc_news:/tv/tv_program $1 /en/news_presenter:/tv/non_character_role $2)))))"); + else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (/organization/leadership@person@title@as_of_date@organization@role@from@to:t $0 /en/chief_executive_officer:/type/text $1 /en/apple_inc:/organization/organization $2 $3 $4))))))")) + questionWriter.println("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (exists $4 (/organization/leadership@person@title@as_of_date@organization@role@from@to:t $0 $1 $2 /en/apple_inc:/organization/organization $3 /en/chief_executive_officer:/type/text $4 $5))))))"); + else if (line.contains("/en/the_nutty_professor")) { + String replaceLine = line.replace("/en/the_nutty_professor", "/en/the_nutty_professor_1996"); + questionWriter.println(replaceLine); + } else if (line.contains("/base/boxing")) { + String replaceLine = line.replace("/base/boxing", "/boxing"); + questionWriter.println(replaceLine); + } else questionWriter.println(line); + } + questionWriter.close(); + + PrintWriter npWriter = IOUtils.getPrintWriter(outNpFile); + for (String line : IOUtils.readLines(inNpFile)) { + if (line.equals("firefly :- NP : /base/ranker/rankerurlname/firefly$002F143400:/tv/tv_program")) + npWriter.println("firefly :- NP : /m/014v3t:/tv/tv_program"); + else if (line.equals("manhattan bridge :- NP : /en/suspension_bridge:/transportation/bridge")) + npWriter.println("manhattan bridge :- NP : /en/manhattan_bridge:/transportation/bridge"); + else if (line.equals("the beastie boys :- NP : /m/0116j8:/music/track")) + npWriter.println("the beastie boys :- NP : /m/0116j8:/music/recording"); + else if (line.equals("sabotage :- NP : /m/0l16j8:/music/track")) + npWriter.println("sabotage :- NP : /m/0l16j8:/music/recording"); + else if (line.equals("travels with my cello :- NP : /m/03crp32:/book/written_work")) + npWriter.println("travels with my cello :- NP : /m/067y_k7:/book/written_work"); + else if (line.equals("film actor :- NP : /film/film_actor:/freebase/type_profile")) + npWriter.println("film actor :- NP : /film/actor:/freebase/type_profile"); + else if (line.equals("invertigo :- NP : /m/03mfjrv:/amusement_parks/ride")) + npWriter.println("invertigo :- NP : /m/0flmt0:/amusement_parks/ride"); + else if (line.equals("ceratopsia :- NP : /en/ceratopsia/-/base/dinosaur:/base/dinosaur/dinosaur")) + npWriter.println("ceratopsia :- NP : /en/ceratopsia:/base/dinosaur/dinosaur"); + else if (line.equals("barosaurus :- NP : /en/barosaurus/-/base/dinosaur:/base/dinosaur/dinosaur")) + npWriter.println("barosaurus :- NP : /en/barosaurus:/base/dinosaur/dinosaur"); + else if (line.equals("mercury :- NP : /authority/us/gov/hhs/fda/srs-unii/FXS1BY2PGL:/chemistry/chemical_element")) + npWriter.println("mercury :- NP : /m/025sw5g:/chemistry/chemical_element"); + else if (line.equals("knickerbockers :- NP : /fashion/garment:/fashion/garment")) + npWriter.println("knickerbockers :- NP : /en/knickerbockers:/fashion/garment"); + else if (line.equals("yahoo! :- NP : /business/cik/0001011006:/organization/organization")) + npWriter.println("yahoo! :- NP : /m/019rl6:/organization/organization"); + else if (line.equals("uss alabama :- NP : /base/usnris/item/86000083:/location/location")) + npWriter.println("uss alabama :- NP : /m/019zhn:/location/location"); + else if (line.equals("lohengrin :- NP : /base/imslp/65847:/opera/opera")) + npWriter.println("lohengrin :- NP : /m/09hvx:/opera/opera"); + else if (line.equals("alice in wonderland :- NP : /source/allocine/fr/film/132663:/film/film")) + npWriter.println("alice in wonderland :- NP : /m/04jpg2p:/film/film"); + else if (line.equals("bauhaus :- NP : /en/bauhaus:/architecture/architectural_style")) + npWriter.println("bauhaus :- NP : /en/international_style:/architecture/architectural_style"); + else if (line.equals("polaris :- NP : /en/polaris:/astronomy/star")) + npWriter.println("polaris :- NP : /m/0kjyrc7:/astronomy/star"); + else if (line.equals("mitchell public library :- NP : /en/mitchell_public_library:/library/public_library")) + npWriter.println("mitchell public library :- NP : /m/0j9by57:/library/public_library"); + else if (line.equals("gold :- NP : /quotationsbook/subject/gold:/chemistry/chemical_element")) + npWriter.println("gold :- NP : /m/025rs2z:/chemistry/chemical_element"); + else if (line.equals("nutty professor :- NP : /en/the_nutty_professor:/film/film")) + npWriter.println("nutty professor :- NP : /en/the_nutty_professor_1996:/film/film"); + else if (line.equals("film domain :- NP : /film:/freebase/domain_profile")) + npWriter.println("film domain :- NP : /m/010s:/freebase/domain_profile"); + else if (line.equals("newscaster :- NP : /en/newscaster:/tv/non_character_role")) + npWriter.println("newscaster :- NP : /en/news_presenter:/tv/non_character_role"); + else if (line.equals("ali�frazier ii :- NP : /en/ali-frazier_ii:/base/boxing/boxing_match")) { + } else + npWriter.println(line); + } + npWriter.println("the battle of the champions :- NP : /m/0kvlz:/boxing/boxing_match"); + npWriter.println("wba world champion :- NP : /m/0chgh2j:/boxing/boxing_title"); + npWriter.println("muhammad ali vs. joe frazier ii :- NP : /en/ali-frazier_ii:/boxing/boxing_match"); + npWriter.close(); + } + + // TODO - handle all entities that do not start with fb:m. or fb:en. + private void convertExampleFile(String inFile, String outPrefix) throws IOException { + + PrintWriter formulaWriter = IOUtils.getPrintWriter(outPrefix + ".formulas"); + BufferedReader reader = IOUtils.getBufferedFileReader(inFile); + List examples = new ArrayList(); + String line = reader.readLine(); + while (line != null) { + Example example = new Example.Builder() + .setUtterance(line) + .setTargetFormula(processFree917LogicalForm(reader.readLine())) + .createExample(); + line = reader.readLine(); + line = reader.readLine(); + examples.add(example.toJson()); + formulaWriter.println(example.targetFormula); + } + LogInfo.log("Arg count distribution: " + argnumCounter); + reader.close(); + formulaWriter.close(); + + + int split = (int) (0.7 * examples.size()); + int[] perm = SampleUtils.samplePermutation(new Random(1), examples.size()); + List train = new ArrayList(); + List test = new ArrayList(); + for (int i = 0; i < split; i++) + train.add(examples.get(perm[i])); + for (int i = split; i < examples.size(); i++) + test.add(examples.get(perm[i])); + printToFile(outPrefix + ".train.examples", train); + printToFile(outPrefix + ".test.examples", test); + printToFile(outPrefix + ".examples", examples); + } + + private void printToFile(String fileName, List examples) throws IOException { + PrintWriter exampleWriter = IOUtils.getPrintWriter(fileName); + for (String example : examples) { + exampleWriter.println(example); + } + exampleWriter.close(); + } + + private Formula processFree917LogicalForm(String free917LogicalForm) { + + LispTree tree = LispTree.proto.parseFromString(free917LogicalForm); + if (tree.child(0).value.equals("lambda")) { + // error check + if (!tree.child(1).value.equals("$0") || tree.children.size() != 4) + throw new RuntimeException("Illegal lambda expression: " + free917LogicalForm); + return handleLambda(tree); + } else if (tree.child(0).value.equals("count")) { + if (!tree.child(1).value.equals("$0") || tree.children.size() != 3) + throw new RuntimeException("Illegal lambda expression: " + free917LogicalForm); + return handleCount(tree); + } else if (tree.child(0).value.startsWith("/")) { + return handleAsk(tree); + } else + throw new RuntimeException("Unknown free917 logical form: " + free917LogicalForm); + } + + private Formula handleLambda(LispTree tree) { + return handleBody(tree.child(3)); + } + + private Formula handleCount(LispTree tree) { + Formula formula = handleBody(tree.child(2)); + if (formula == null) + return null; + return new AggregateFormula(AggregateFormula.Mode.count, formula); + } + + private Formula handleBody(LispTree tree) { + Map argToPredMap = new HashMap(); + handleBodyRecurse(tree, argToPredMap); + return generateFormula(argToPredMap); + } + + private Formula generateFormula(Map argToPredMap) { + + if (argToPredMap.size() == 1) { + String arg = argToPredMap.keySet().iterator().next(); + String pred = argToPredMap.get(arg); + + BinaryFormulaInfo info = formulaInfo.getBinaryInfo(Formulas.fromLispTree(LispTree.proto.parseFromString(pred))); + if (info != null) { + String type = formulaInfo.getBinaryInfo(Formulas.fromLispTree(LispTree.proto.parseFromString(pred))).expectedType1; + if (cvts.contains(type)) { + return fixCvtFormulas(pred, arg); + } + } + + return new JoinFormula(pred, getArgFormula(arg)); + } else { + if (argToPredMap.get("target") == null) + throw new RuntimeException("target is null: " + argToPredMap); + + Formula argsFormula = conjunctArgs(argToPredMap); + Formula targetFormula = Formulas.reverseFormula( + new ValueFormula(new NameValue(argToPredMap.get("target")))); + Formula res = new JoinFormula(targetFormula, argsFormula); + return res; + } + } + + private Formula fixCvtFormulas(String pred, String arg) { + + Formula join = new JoinFormula(pred, getArgFormula(arg)); + if (pred.equals("!fb:automotive.trim_level.msrp") || pred.equals("!fb:event.disaster.damage") || + pred.equals("!fb:comic_books.comic_book_issue.cover_price")) { + return new JoinFormula("!fb:measurement_unit.money_value.amount", join); + } else if (pred.equals("!fb:celebrities.celebrity.net_worth") || pred.equals("!fb:projects.project.actual_cost") + || pred.equals("!fb:digicams.digital_camera.street_price") || pred.equals("!fb:amusement_parks.ride.cost")) { + return new JoinFormula("!fb:measurement_unit.dated_money_value.amount", join); + } else if (pred.equals("!fb:computer.software.compatible_oses")) { + return new JoinFormula("!fb:computer.software_compatibility.operating_system", join); + } else if (pred.equals("!fb:finance.stock_exchange.companies_traded")) { + return new JoinFormula("!fb:business.stock_ticker_symbol.ticker_symbol", join); + } else if (pred.equals("!fb:medicine.hospital.beds") || pred.equals("!fb:metropolitan_transit.transit_system.daily_riders") + || pred.equals("!fb:library.public_library_system.collection_size") || pred.equals("!fb:library.public_library_system.annual_visits") + || pred.equals("!fb:protected_sites.protected_site.annual_visitors") || pred.equals("!fb:amusement_parks.park.annual_visits") + || pred.equals("!fb:education.educational_institution.total_enrollment") + || pred.equals("!fb:religion.religion.number_of_adherents")) { + return new JoinFormula("!fb:measurement_unit.dated_integer.number", join); + } else if (pred.equals("!fb:business.employer.employees")) { + return new JoinFormula("!fb:business.employment_tenure.person", join); + } else if (pred.equals("!fb:tv.tv_series_episode.producers")) { + return new JoinFormula("!fb:tv.tv_producer_episode_credit.producer", join); + } else if (pred.equals("!fb:book.periodical.frequency_or_issues_per_year")) { + return new JoinFormula("!fb:book.periodical_frequency.issues_per_year", join); + } else if (pred.equals("!fb:book.periodical.first_issue_date")) { + return new JoinFormula("!fb:book.periodical_publication_date.date", join); + } else if (pred.equals("!fb:games.game.number_of_players") || pred.equals("!fb:aviation.aircraft_model.passengers")) { + return new JoinFormula("!fb:measurement_unit.integer_range.high_value", join); + } else if (pred.equals("!fb:military.armed_force.personnel")) { + return new JoinFormula("!fb:military.military_service.military_person", join); + } else if (pred.equals("!fb:business.consumer_company.products")) { + return new JoinFormula("!fb:business.company_product_relationship.consumer_product", join); + } else if (pred.equals("!fb:location.location.geolocation")) { + return new JoinFormula("!fb:location.geocode.longitude", join); + } + return new JoinFormula(pred, getArgFormula(arg)); + + } + + private Formula conjunctArgs(Map argToPredMap) { + + List pivots = new ArrayList(); + for (String arg : argToPredMap.keySet()) { + if (!arg.equals("target")) + pivots.add(constructJoin(arg, argToPredMap.get(arg))); + } + Formula res = pivots.get(0); + if (pivots.size() == 1) return res; + for (int i = 1; i < pivots.size(); ++i) + res = new MergeFormula(Mode.and, res, pivots.get(i)); + return res; + } + + private Formula getArgFormula(String arg) { + if (arg.startsWith("fb:")) + return new ValueFormula(new NameValue(arg)); + if (arg.startsWith("DATE::")) { + String[] tokens = arg.split("::"); + return new ValueFormula(DateValue.parseDateValue(tokens[1])); + } + // TODO make sure ints and booleans work + if (arg.startsWith("INT::")) { + String[] tokens = arg.split("::"); + return new ValueFormula(new NumberValue(Double.parseDouble(tokens[1]), NumberValue.unitless)); + } + if (arg.startsWith("BOOL::")) { + String[] tokens = arg.split("::"); + return new ValueFormula(new NameValue(tokens[1])); + } + if (arg.startsWith("TEXT::")) { + String[] tokens = arg.split("::"); + return new ValueFormula(new StringValue(tokens[1])); + } + throw new RuntimeException("Unknown arg: " + arg); + } + + private JoinFormula constructJoin(String arg, String pred) { + VariableFormula var = new VariableFormula("x"); + JoinFormula join = new JoinFormula(pred, var); + LambdaFormula lambda = new LambdaFormula("x", join); + return new JoinFormula(lambda, getArgFormula(arg)); + } + + private void handleBodyRecurse(LispTree tree, Map argToPredMap) { + if (tree.child(0).value.equals("exists")) { + if (tree.children.size() != 3) + throw new RuntimeException("bad exists clause: " + tree); + handleBodyRecurse(tree.child(2), argToPredMap); + } else { + if (!tree.child(0).value.startsWith("/")) + throw new RuntimeException("bad exists clause: " + tree); + + // parse the relation + + String predicate = tree.child(0).value; + String[] predTokens = predicate.substring(0, predicate.lastIndexOf(':')).split("@"); + if (predTokens.length <= 1) + throw new RuntimeException("Bad body: " + tree); + + String fbType = constructFbType(predTokens[0]); + List fbRelations = constructFbRelations(predTokens, fbType); + + // parse the arguments + if (predTokens.length == 2) { + if (tree.child(1).value.equals("$0")) + argToPredMap.put(parseEntity(tree.child(2).value), fbRelations.get(0)); + else if (tree.child(2).value.equals("$0")) + argToPredMap.put(parseEntity(tree.child(1).value), "!" + fbRelations.get(0)); + else throw new RuntimeException("bad non-cvt tree: " + tree); + } else { + List fbArguments = constructFbArguments(tree); + for (int i = 0; i < fbArguments.size(); ++i) { + String fbRelation = fbRelations.get(i); + if (fbArguments.get(i).equals("$0")) + argToPredMap.put("target", fbRelation); + else if (!fbArguments.get(i).startsWith("$")) { + argToPredMap.put(fbArguments.get(i), fbRelation); + } + } + } + argnumCounter.incrementCount(argToPredMap.size()); + } + } + + private List constructFbArguments(LispTree tree) { + boolean lastName = false; + List res = new ArrayList(); + for (int j = 1; j < tree.children.size(); ++j) { + + if (tree.child(j).value.equals("$0")) { + res.add("$0"); + lastName = false; + } else if (tree.child(j).value.startsWith("/")) { + String entity = parseEntity(tree.child(j).value); + res.add(entity); + lastName = true; + } else { + if (!lastName) + res.add(tree.child(j).value); + lastName = false; + } + } + return res; + } + + private List constructFbRelations(String[] predTokens, String fbType) { + + List res = new ArrayList(); + if (predTokens[0].contains("&")) { + String relation = "!" + FormatConverter.fromSlashToDot(predTokens[0].substring(0, predTokens[0].indexOf('&')), true); + res.add(relation); + } + for (int j = 1; j < predTokens.length; ++j) + res.add(fbType + "." + predTokens[j]); + return res; + } + + private String constructFbType(String predHead) { + if (predHead.contains("&")) + return FormatConverter.fromSlashToDot(predHead.substring(predHead.indexOf('&') + 1), true); + return FormatConverter.fromSlashToDot(predHead, true); + + } + + private String parseEntity(String value) { + + if (value.startsWith("/type/datetime/")) { + String[] tokens = value.split(":"); + String date = tokens[0].substring(tokens[0].lastIndexOf('/') + 1); + return "DATE::" + date; + } else if (value.startsWith("/type/boolean/")) { + String[] tokens = value.split(":"); + String b = tokens[0].substring(tokens[0].lastIndexOf('/') + 1); + return "BOOL::" + b; + } else if (value.startsWith("/type/int/")) { + String[] tokens = value.split(":"); + String i = tokens[0].substring(tokens[0].lastIndexOf('/') + 1); + return "INT::" + i; + } else if (value.startsWith("/type/text/")) { + String[] tokens = value.split(":"); + String i = tokens[0].substring(tokens[0].lastIndexOf('/') + 1); + return "TEXT::" + i; + } + return FormatConverter.fromSlashToDot(value.substring(0, value.indexOf(':')), true); + } + + /** There is one example and this method is tailored for that */ + private Formula handleAsk(LispTree tree) { + String pred = tree.child(0).value; + ValueFormula arg1 = Formulas.newNameFormula(parseEntity(tree.child(1).value)); + ValueFormula arg2 = Formulas.newNameFormula(parseEntity(tree.child(2).value)); + pred = pred.replace('@', '/'); + pred = pred.substring(0, pred.lastIndexOf(':')); + pred = FormatConverter.fromSlashToDot(pred, true); + ValueFormula predFormula = Formulas.newNameFormula(pred); + JoinFormula pf = new JoinFormula(predFormula, arg2); + MergeFormula mf = new MergeFormula(Mode.and, arg1, pf); + return mf; + } + + public void genreateEntityInfoFile(String free917EntityFile, String entityInfoFile, String outFile, String missingEntitiesFile) throws IOException { + + Map midToIdMap = FileUtils.loadStringToStringMap(midToIdFile); + + Map> idToNameMap = new HashMap>(); + for (String line : IOUtils.readLines(free917EntityFile)) { + + String[] tokens = line.split(":-"); + String name = tokens[0].trim().replace('-', ' '); + name = name.replace("#", "# "); + name = name.replace("!", " !"); + name = name.replace("'", " '"); + name = name.replace(",", " ,"); + name = name.replace(":", " :"); + String entry = tokens[1].trim(); + String[] entryTokens = entry.split(":"); + String id = entryTokens[1].trim(); + id = FormatConverter.fromSlashToDot(id, true); + + if (midToIdMap.containsKey(id)) + id = midToIdMap.get(id); + + MapUtils.add(idToNameMap, id, name); + } + LogInfo.log("Number of entries: " + idToNameMap.size()); + + PrintWriter writer = IOUtils.getPrintWriter(outFile); + int i = 0; + for (String line : IOUtils.readLines(entityInfoFile)) { + String[] tokens = line.split("\t"); + String id = tokens[1]; + + if (idToNameMap.containsKey(id)) { + + if (idToNameMap.get(id).size() > 1) + System.out.println("Multiple names: " + idToNameMap.get(id)); + for (String name : idToNameMap.get(id)) { + if (name.equals("beer")) { + tokens[3] = "beer"; + writer.println(StringUtils.join(tokens, "\t")); + tokens[3] = "beers"; + writer.println(StringUtils.join(tokens, "\t")); + } else if (name.equals("film actor")) { + tokens[3] = "film actor"; + writer.println(StringUtils.join(tokens, "\t")); + tokens[3] = "film actors"; + writer.println(StringUtils.join(tokens, "\t")); + } else { + tokens[3] = name; + writer.println(StringUtils.join(tokens, "\t")); + } + } + idToNameMap.remove(id); + } + if (i % 1000000 == 0) + System.out.println("Lines: " + i++); + i++; + } + writer.close(); + PrintWriter missingWriter = IOUtils.getPrintWriter(missingEntitiesFile); + for (String id : idToNameMap.keySet()) { + if (id.startsWith("fb:type") || id.startsWith("fb:un.")) + continue; + missingWriter.println(id + "\t" + idToNameMap.get(id)); + } + missingWriter.close(); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/FreebaseInfo.java b/src/edu/stanford/nlp/sempre/freebase/FreebaseInfo.java new file mode 100644 index 0000000..d5ba121 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/FreebaseInfo.java @@ -0,0 +1,293 @@ +package edu.stanford.nlp.sempre.freebase; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; + +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.BinaryFormulaInfo; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.UnaryFormulaInfo; +import edu.stanford.nlp.sempre.*; +import fig.basic.*; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.*; + +/** + * Class for keeping info from Freebase schema + * @author jonathanberant + */ +public final class FreebaseInfo { + private static FreebaseInfo singleton; + public static FreebaseInfo getSingleton() { + if (singleton == null) singleton = new FreebaseInfo(); + return singleton; + } + + public static class Options { + @Option(gloss = "ttl file with schema information") + public String schemaPath = "lib/fb_data/93.exec/schema2.ttl"; + } + public static Options opts = new Options(); + + // any + // - number (boolean, int, float, date) + // - text + // - entity (people, loc, org, ...) + // - cvt + + // Concrete primitive types + public static final String BOOLEAN = CanonicalNames.BOOLEAN; + public static final String INT = CanonicalNames.INT; + public static final String FLOAT = CanonicalNames.FLOAT; + public static final String DATE = CanonicalNames.DATE; + public static final String TEXT = CanonicalNames.TEXT; + public static final String NUMBER = CanonicalNames.NUMBER; + public static final String ENTITY = CanonicalNames.ENTITY; + public static final String ANY = CanonicalNames.ANY; + + // Common entity types + public static final String PERSON = "fb:people.person"; + + // Non-standard abstract types + public static final String CVT = "fb:type.cvt"; + + // Common relations + public static final String TYPE = CanonicalNames.TYPE; + public static final String NAME = CanonicalNames.NAME; + public static final String PROF = "fb:people.person.profession"; + public static final String ALIAS = "fb:common.topic.alias"; + + // mapping from master property to its opposite (e.g., fb:people.person.place_of_birth => fb:location.location.people_born_here) + private BiMap masterToOppositeMap = HashBiMap.create(); + + private Set cvts = new HashSet<>(); + private Map type1Map = new HashMap<>(); // property => type of arg1 + private Map type2Map = new HashMap<>(); // property => type of arg2 + private Map unit2Map = new HashMap<>(); // property => unit of arg2 (if exists) + private Map> bDescriptionsMap = new HashMap<>(); // property => descriptions + private Map bPopularityMap = new HashMap<>(); // property => popularity + // unary maps + private Map professionPopularityMap = new HashMap<>(); // property => popularity + private Map typePopularityMap = new HashMap<>(); // property => popularity + private Map> professionDescriptionsMap = new HashMap<>(); // property => descriptions + private Map> typeDescriptionsMap = new HashMap<>(); // property => descriptions + + private Map nameMap = new HashMap(); // id => name of id + + public String getArg1Type(String property) { return type1Map.get(property); } + public String getArg2Type(String property) { return type2Map.get(property); } + + private FreebaseInfo() { + try { + readSchema(); + } catch (NumberFormatException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Go over schema twice - once to populate all fields except descriptions, the second time we populate descriptions after we now what + * are the properties we are interested in + * @throws NumberFormatException + * @throws IOException + */ + public void readSchema() throws IOException { + LogInfo.begin_track("Loading Freebase schema: %s", opts.schemaPath); + BufferedReader in = IOUtils.openInHard(opts.schemaPath); + + // Include mediator types + SemTypeHierarchy.singleton.addSupertype(CVT, CVT); + SemTypeHierarchy.singleton.addSupertype(CVT, ANY); + + String line; + while ((line = in.readLine()) != null) { + String[] tokens = edu.stanford.nlp.sempre.freebase.Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + String property = tokens[1]; + String arg2 = tokens[2]; + + if (property.equals("fb:type.property.reverse_property")) { // reverse_property => opposite_property + // Duplicates logically really shouldn't happen but the Freebase RDF + // reverse properties are not 1:1. We should monitor this and make + // sure we don't lose any alignments. + if (masterToOppositeMap.containsKey(arg1)) { + // LogInfo.errors("arg1 exists multiple times: %s", line); + continue; + } + if (masterToOppositeMap.inverse().containsKey(arg2)) { + // LogInfo.errors("arg2 exists multiple times: %s", line); + continue; + } + masterToOppositeMap.put(arg1, arg2); + } else if (property.equals("fb:freebase.type_hints.included_types")) { // included_types => supertypes + SemTypeHierarchy.singleton.addSupertype(arg1, arg2); + SemTypeHierarchy.singleton.addEntitySupertypes(arg1); + SemTypeHierarchy.singleton.addEntitySupertypes(arg2); + } else if (property.equals("fb:freebase.type_hints.mediator")) { // mediator => cvt + if (arg2.equals("\"true\"^^xsd:boolean")) cvts.add(arg1); + else if (arg2.equals("\"false\"^^xsd:boolean")) cvts.remove(arg1); + else throw new RuntimeException("Invalid xsd:boolean: " + arg2); + } else if (property.equals("fb:type.property.schema")) { // schema => type1 + if (type1Map.containsKey(arg1)) + LogInfo.errors("%s already has type1 %s, assigning %s", arg1, type1Map.get(arg1), arg2); + type1Map.put(arg1, arg2); + } else if (property.equals("fb:type.property.expected_type")) { // expected_type => type2 + if (type2Map.containsKey(arg1)) + LogInfo.errors("%s already has type2 %s, assigning %s", arg1, type2Map.get(arg1), arg2); + type2Map.put(arg1, arg2); + } else if (property.equals("fb:type.property.unit")) { + unit2Map.put(arg1, arg2); + } else if (property.equals("fb:user.custom.type.property.num_instances")) { + bPopularityMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseInt(arg2)); + } else if (property.equals("fb:user.custom.people.person.profession.num_instances")) { + professionPopularityMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseInt(arg2)); + } else if (property.equals("fb:user.custom.type.object.type.num_instances")) { + typePopularityMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseInt(arg2)); + } + } + in.close(); + + // Second iteration - populate descriptions assumes all properties have the fb:type.property.num_instances field + in = IOUtils.openInHard(opts.schemaPath); + while ((line = in.readLine()) != null) { + String[] tokens = edu.stanford.nlp.sempre.freebase.Utils.parseTriple(line); + if (tokens == null) continue; + String arg1 = tokens[0]; + String property = tokens[1]; + String arg2 = tokens[2]; + + if (property.equals(NAME) || property.equals(ALIAS)) { + if (bPopularityMap.containsKey(arg1)) { + MapUtils.addToList(bDescriptionsMap, arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2).toLowerCase()); + } else if (professionPopularityMap.containsKey(arg1)) { + MapUtils.addToList(professionDescriptionsMap, arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2).toLowerCase()); + } else if (typePopularityMap.containsKey(arg1)) { + MapUtils.addToList(typeDescriptionsMap, arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2).toLowerCase()); + } + } + + if (property.equals(NAME)) + nameMap.put(arg1, edu.stanford.nlp.sempre.freebase.Utils.parseStr(arg2)); + } + LogInfo.logs("%d CVTs, (%d,%d) property types, %d property units", cvts.size(), type1Map.size(), type2Map.size(), unit2Map.size()); + LogInfo.end_track(); + } + + public Map createBinaryFormulaInfoMap() { + + Map res = new HashMap<>(); + for (String property : bPopularityMap.keySet()) { + Formula f = Formulas.fromLispTree(LispTree.proto.parseFromString(property)); + BinaryFormulaInfo info = new BinaryFormulaInfo(f, type1Map.get(property), type2Map.get(property), unit2Map.get(property), "", bDescriptionsMap.get(property), bPopularityMap.get(property)); + if (!info.isComplete()) { + continue; + } + res.put(f, info); + } + return res; + } + + public Map createUnaryFormulaInfoMap() { + + Map res = new HashMap(); + // professions + for (String profession : professionPopularityMap.keySet()) { + Formula f = new JoinFormula(PROF, new ValueFormula(new NameValue(profession))); + UnaryFormulaInfo info = new UnaryFormulaInfo(f, professionPopularityMap.get(profession), + MapUtils.get(professionDescriptionsMap, profession, new LinkedList()), + Collections.singleton(PERSON)); + if (!info.isComplete()) { + continue; + } + res.put(f, info); + } + // types + for (String type : typePopularityMap.keySet()) { + Formula f = new JoinFormula(TYPE, new ValueFormula(new NameValue(type))); + UnaryFormulaInfo info = new UnaryFormulaInfo(f, typePopularityMap.get(type), + MapUtils.get(typeDescriptionsMap, type, new LinkedList()), + Collections.singleton(type)); + if (!info.isComplete()) { + continue; + } + res.put(f, info); + } + return res; + } + + // fb:people.person.place_of_birth => true + public boolean propertyHasOpposite(String property) { + return masterToOppositeMap.containsKey(property) || masterToOppositeMap.inverse().containsKey(property); + } + // fb:people.person.place_of_birth => fb:location.location.people_born_here + public String getOppositeFbProperty(String property) { + if (masterToOppositeMap.containsKey(property)) + return masterToOppositeMap.get(property); + if (masterToOppositeMap.inverse().containsKey(property)) + return masterToOppositeMap.inverse().get(property); + throw new RuntimeException("Property does not have an opposite: " + property); + } + + public String getUnit1(String property) { return typeToUnit(type1Map.get(property), property); } + public String getUnit2(String property) { return typeToUnit(type2Map.get(property), property); } + + // Get the measurement unit associated with this type. + // If something is not a number, then return something crude (e.g. fb:type.cvt). + // Return null if we don't know anything. + public String typeToUnit(String type, String property) { + if (type == null) { + // LogInfo.errors("No type information for property: %s", property); + return null; + } + if (type.equals(INT) || type.equals(FLOAT)) { + String unit = unit2Map.get(property); + if (unit == null) { + // LogInfo.errors("No unit information for property: %s", property); + return NumberValue.unitless; + } + return unit; + } + if (type.equals(BOOLEAN) || type.equals(TEXT) || type.equals(DATE)) // Use the type as the unit + return type; + if (isCvt(type)) return CVT; // CVT + return ENTITY; // Entity + } + + public boolean isCvt(String type) { + return cvts.contains(type); + } + + public String getPropertyName(String property) { + List names = bDescriptionsMap.get(property); + if (names == null) return null; + return names.get(0); + } + + public String getName(String id) { return nameMap.get(id); } + + public static boolean isReverseProperty(String property) { + return CanonicalNames.isReverseProperty(property); + } + public static String reverseProperty(String property) { + return CanonicalNames.reverseProperty(property); + } + + // fb:en.barack_obama => http://rdf.freebase.com/ns/en/barack_obama + public static final String freebaseNamespace = "http://rdf.freebase.com/ns/"; + + public static String id2uri(String id) { + assert id.startsWith("fb:") : id; + return freebaseNamespace + id.substring(3).replaceAll("\\.", "/"); + } + public static String uri2id(String uri) { + if (!uri.startsWith(freebaseNamespace)) { + LogInfo.logs("Warning: invalid Freebase uri: %s", uri); + // Don't do any conversion; this is not necessarily the best thing to do. + return uri; + } + return "fb:" + uri.substring(freebaseNamespace.length()).replaceAll("/", "."); + } +} diff --git a/src/edu/stanford/nlp/sempre/FreebaseSearch.java b/src/edu/stanford/nlp/sempre/freebase/FreebaseSearch.java similarity index 79% rename from src/edu/stanford/nlp/sempre/FreebaseSearch.java rename to src/edu/stanford/nlp/sempre/freebase/FreebaseSearch.java index f2cd9b1..869d055 100644 --- a/src/edu/stanford/nlp/sempre/FreebaseSearch.java +++ b/src/edu/stanford/nlp/sempre/freebase/FreebaseSearch.java @@ -1,5 +1,8 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.freebase; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.cache.StringCache; +import edu.stanford.nlp.sempre.cache.StringCacheUtils; import fig.basic.*; import java.io.*; import java.net.SocketTimeoutException; @@ -34,7 +37,7 @@ public class FreebaseSearch { tree.addChild(mid); tree.addChild(id == null ? "" : id); tree.addChild(name); - tree.addChild(score+""); + tree.addChild(score + ""); return tree; } @@ -48,6 +51,9 @@ public class FreebaseSearch { @Option(gloss = "Milliseconds to wait until reading connection times out") public int readTimeoutMs = 1 * 60 * 1000; + @Option(gloss = "API key (needed to get more access)") + public String apiKey; + @Option(gloss = "Save results of Freebase API search") public String cachePath; } @@ -56,10 +62,11 @@ public class FreebaseSearch { private final StringCache cache; public class ServerResponse { - public ServerResponse() { this.entries = new ArrayList(); this.error = null; } + public ServerResponse() { this.entries = new ArrayList<>(); this.error = null; } public ServerResponse(ErrorValue error) { this.entries = null; this.error = error; } public final List entries; public final ErrorValue error; + boolean cached; long timeMs; } @@ -78,14 +85,17 @@ public class FreebaseSearch { ServerResponse response = new ServerResponse(); // First, try the cache. - if (cache != null) + if (cache != null) { output = cache.get(query); + if (output != null) response.cached = true; + } // If got nothing, then need to hit the server. if (output == null) { try { // Setup the connection String url = String.format("https://www.googleapis.com/freebase/v1/search?query=%s", URLEncoder.encode(query, "UTF-8")); + if (opts.apiKey != null) url += "&key=" + opts.apiKey; URLConnection conn = new URL(url).openConnection(); conn.setConnectTimeout(opts.connectTimeoutMs); conn.setReadTimeout(opts.readTimeoutMs); @@ -118,20 +128,20 @@ public class FreebaseSearch { // Parse the result Map results = Json.readMapHard(output); - for (Object resultObj : (List)results.get("result")) { - Map result = (Map)resultObj; - //LogInfo.logs("FreebaseSearch result: %s", result); - String mid = (String)result.get("mid"); - String id = (String)result.get("id"); + for (Object resultObj : (List) results.get("result")) { + Map result = (Map) resultObj; + String mid = (String) result.get("mid"); + String id = (String) result.get("id"); mid = toRDF(mid); id = toRDF(id); - String name = (String)result.get("name"); - double score = (double)result.get("score"); + String name = (String) result.get("name"); + double score = (double) result.get("score"); response.entries.add(new Entry(mid, id, name, score)); } watch.stop(); response.timeMs = watch.getCurrTimeLong(); + LogInfo.logs("FreebaseSearch %s => %s results (cached=%s)", query, response.entries.size(), response.cached); return response; } @@ -142,9 +152,9 @@ public class FreebaseSearch { } public static void main(String[] args) { - //opts.cachePath = "FreebaseSearch.cache"; + opts.cachePath = "FreebaseSearch.cache"; String query = StrUtils.join(args, " "); - //query = "obama"; + query = "obama"; FreebaseSearch search = new FreebaseSearch(); LogInfo.logs("%s", search.lookup(query).entries); } diff --git a/src/edu/stanford/nlp/sempre/freebase/FreebaseTypeLookup.java b/src/edu/stanford/nlp/sempre/freebase/FreebaseTypeLookup.java new file mode 100644 index 0000000..5c6df80 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/FreebaseTypeLookup.java @@ -0,0 +1,64 @@ +package edu.stanford.nlp.sempre.freebase; + +import java.util.*; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.cache.*; +import fig.basic.*; + +/** + * Provides types of Freebase entities and properties. + * For entities, look them up (requires access to the cache file). + * For properties, just look them up in the FreebaseInfo schema. + */ +public class FreebaseTypeLookup implements TypeLookup { + public static class Options { + @Option(gloss = "Cache path to the types path") + public String entityTypesPath = "localhost:4000:/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized.en-types"; + } + public static Options opts = new Options(); + + // Given those ids, we retrieve the set of types + private static StringCache entityTypesCache; + + public Set getEntityTypes(String entity) { + if (opts.entityTypesPath == null) + return Collections.singleton(FreebaseInfo.ENTITY); + + // Read types from cache + if (entityTypesCache == null) entityTypesCache = StringCacheUtils.create(opts.entityTypesPath); + Set types = new HashSet<>(); + String typesStr = entityTypesCache.get(entity); + if (typesStr != null) { + Collections.addAll(types, typesStr.split(",")); + } else { + types.add(FreebaseInfo.ENTITY); + } + return types; + } + + @Override + public SemType getEntityType(String entity) { + Set types = getEntityTypes(entity); + // Remove supertypes + // TODO(pliang): this is inefficient! + Set resultTypes = new HashSet<>(types); + for (String entityType : types) { + for (String supertype : SemTypeHierarchy.singleton.getSupertypes(entityType)) { + if (!supertype.equals(entityType)) + resultTypes.remove(supertype); + } + } + return SemType.newUnionSemType(resultTypes); + } + + @Override + public SemType getPropertyType(String property) { + // property = fb:location.location.area + // arg1Type = fb:location.location --> becomes retType (head of formula) + // arg2Type = fb:type.float --> becomes argType + FreebaseInfo info = FreebaseInfo.getSingleton(); + String arg1Type = info.getArg1Type(property), arg2Type = info.getArg2Type(property); + if (arg1Type == null || arg2Type == null) return null; + return SemType.newFuncSemType(arg2Type, arg1Type); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/FreebaseValueEvaluator.java b/src/edu/stanford/nlp/sempre/freebase/FreebaseValueEvaluator.java new file mode 100644 index 0000000..6d04b49 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/FreebaseValueEvaluator.java @@ -0,0 +1,92 @@ +package edu.stanford.nlp.sempre.freebase; + +import fig.basic.*; +import java.util.List; +import edu.stanford.nlp.sempre.*; + +/** + * Used to evaluate Freebase question answering. + * Denotation is a list of entities. + * Nothing in here is specific to Freebase, but this is not really meant to be + * a general-purpose class. + * + * @author Percy Liang + */ +public class FreebaseValueEvaluator implements ValueEvaluator { + public static class Options { + @Option(gloss = "When evaluating lists, compute F1 rather than exact match") public boolean useF1 = true; + } + public static final Options opts = new Options(); + + public double getCompatibility(Value target, Value pred) { + double f1 = getF1(target, pred); + return opts.useF1 ? f1 : (f1 == 1 ? 1 : 0); + } + + // Compute F1 score between two lists (partial match). + // this is target, that is predicted. + private double getF1(Value target, Value pred) { + List targetList = ((ListValue) target).values; + if (!(pred instanceof ListValue)) return 0; + List predList = ((ListValue) pred).values; + + if (targetList.size() == 0 && predList.size() == 0) + return 1; + if (targetList.size() == 0 || predList.size() == 0) + return 0; + + double precision = 0; + for (Value v2 : predList) { // For every predicted value... + double score = 0; + for (Value v1 : targetList) + score = Math.max(score, getItemCompatibility(v1, v2)); + precision += score; + } + precision /= predList.size(); + assert precision >= 0 && precision <= 1 : precision; + + double recall = 0; + for (Value v1 : targetList) { // For every true value... + double score = 0; + for (Value v2 : predList) + score = Math.max(score, getItemCompatibility(v1, v2)); + recall += score; + } + recall /= targetList.size(); + assert recall >= 0 && recall <= 1 : recall; + + if (precision + recall == 0) return 0; + + double f1 = 2 * precision * recall / (precision + recall); + assert f1 >= 0 && f1 <= 1 : f1; + + return f1; + } + + // Compare one element of the list. + public double getItemCompatibility(Value target, Value pred) { + if (target instanceof DescriptionValue) { + // Just has to match the description + if (pred instanceof NameValue) + return ((DescriptionValue) target).value.equals(((NameValue) pred).description) ? 1 : 0; + return 0; + } + + if (pred instanceof ErrorValue) return 0; // Never award points for error + if (pred == null) { + LogInfo.warning("Predicted value is null!"); + return 0; + } + if (target.getClass() != pred.getClass()) return 0; + + if (target instanceof DateValue) { + DateValue targetDate = (DateValue) target; + DateValue predDate = (DateValue) pred; + // Only comparing the year right now! This is crude. + boolean match = (targetDate.year == predDate.year); + return match ? 1 : 0; + } + + return target.equals(pred) ? 1 : 0; + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/LambdaCalculusConverter.java b/src/edu/stanford/nlp/sempre/freebase/LambdaCalculusConverter.java new file mode 100644 index 0000000..e436d36 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/LambdaCalculusConverter.java @@ -0,0 +1,544 @@ +package edu.stanford.nlp.sempre.freebase; + +import java.io.*; +import java.util.*; +import fig.basic.*; +import fig.exec.Execution; +import edu.stanford.nlp.sempre.*; + +/** + * Converts Luke Zettlemoyer's lambda calculus data format into our example files. + * @author Percy Liang + * @author Ziang Xie + */ + +public class LambdaCalculusConverter implements Runnable { + public static class Options { + @Option(gloss = "Input path (lambda calculus)") + public String inPath = "freebase/data/geo/geosents600-typed.ccg.dev"; + @Option(gloss = "Specification of translations") + public String specPath = "freebase/data/geo/geo.spec"; + @Option(gloss = "Specification of variable names") + public String varPath = "freebase/data/geo/geo.vars"; + @Option(gloss = "Specification of primitive types") + public String primPath = "freebase/data/geo/geo.primitives"; + @Option(gloss = "Specification of formula replacements") + public String replacePath = "freebase/data/geo/geo.replace"; + @Option(gloss = "Specification of manual conversions") + public String manualConversionsPath = "freebase/data/geo/geo.manual_conversions"; + @Option(gloss = "Input path (examples)") + public String outPath = "freebase/data/geo/geo.out.json"; + @Option(gloss = "Specific example to parse and run") + public int runInd = -1; + @Option(gloss = "Output path for lexicon grammar") + public String lexiconPath = "freebase/data/geo/geo.out.grammar"; + @Option(gloss = "Verbose output (for debugging)") + public boolean verbose = false; // TODO Currently unused + } + public static Options opts = new Options(); + + // Mapping between predicates + Map predicatesMap = new HashMap(); + // Mapping from variables in the input (e.g. "$1") to our variables (e.g. "x") + Map varMap = new HashMap(); + // Mapping from types (e.g. "i" in "population:i") to semparse primitive types + Map primitiveMap = new HashMap(); + // Hardcoded replacements for mis-specified formulas in the input + Map replaceMap = new HashMap(); + // Hardcoded conversions for where the converter fails + Map manualConversionsMap = new HashMap(); + + // Colorize output + Colorizer color = new Colorizer(); + + // Examples to be converted + List examples = new ArrayList(); + // Indices of examples that were executed without error + List validExampleIds = new ArrayList(); + List failedExampleIds = new ArrayList(); + // Use this to later sort the valid examples + List validExampleLengths = new ArrayList(); + + public void run() { + readPrereqs(); + + convertExamples(); + executeExamples(); + + if (opts.runInd < 0) { + writeExamples(); + printSummary(); + } + } + + public void readPrereqs() { + readSpec(); + readPrimitives(); + readVars(); + readStringMap(opts.replacePath, replaceMap); + readStringMap(opts.manualConversionsPath, manualConversionsMap); + } + + void readSpec() { + for (String line : IOUtils.readLinesHard(opts.specPath)) { + if (line.startsWith("#")) continue; + if (line.equals("")) continue; + String[] tokens = line.split(" ", 2); + predicatesMap.put(tokens[0], Formula.fromString(tokens[1])); + } + } + + void readPrimitives() { + for (String line : IOUtils.readLinesHard(opts.primPath)) { + if (line.startsWith("#")) continue; + if (line.equals("")) continue; + String[] tokens = line.split(" ", 2); + primitiveMap.put(tokens[0], tokens[1]); + } + } + + void readVars() { + for (String line : IOUtils.readLinesHard(opts.varPath)) { + if (line.startsWith("#")) continue; + if (line.equals("")) continue; + String[] tokens = line.split(" ", 2); + varMap.put(tokens[0], Formula.fromString(tokens[1])); + } + } + + void readStringMap(String path, Map map) { + boolean readOriginal = false; + String original = ""; + for (String line : IOUtils.readLinesHard(path)) { + if (line.startsWith("#")) continue; + if (line.equals("")) continue; + if (!readOriginal) { + readOriginal = true; + original = line; + } else { + readOriginal = false; + map.put(original, line); + } + } + } + + Formula toPredicate(String func) { + boolean reverse = false; + while (func.startsWith("!")) { + LogInfo.log(func); + reverse = !reverse; + func = func.substring(1); + } + if (!predicatesMap.containsKey(func)) + func = removeGeoType(func); + if (func.startsWith("$")) + return Formula.fromString(String.format("(var %s)", toVariable(func).toString())); + if (!predicatesMap.containsKey(func)) { + throw new RuntimeException("Unknown predicate: " + func); + } + Formula form = predicatesMap.get(func); + if (reverse) { + return new ReverseFormula(form); + } else { + return form; + } + } + + Formula toLambdaVar(String var) { + return toVariable(var); + } + + boolean isVar(String var) { + return var != null && var.startsWith("$"); + } + + Formula toVariable(String var) { + var = removeGeoType(var); + if (!varMap.containsKey(var)) { + throw new RuntimeException("Unknown variable: " + var); + } + return varMap.get(var); + } + + // FIXME Not general to converters + String removeGeoType(String pred) { + return pred.split(":", 2)[0]; + } + + String getGeoType(String pred) { + String[] parts = pred.split(":", 2); + if (parts.length > 0) + return parts[1]; + return ""; + } + + Formula toJoin(String func, Formula arg) { + return new JoinFormula(toPredicate(func), arg); + } + + Formula toLambda(String var, Formula body) { + return new LambdaFormula(toLambdaVar(var).toString(), body); + } + + Formula toMark(String var, Formula body) { + return new MarkFormula(toPredicate(var).toString(), body); + } + + Formula toAndFormula(List clauses, boolean[] hit, String headVar, List existsVars) { + // (and (river:t $1) (loc:t $1 $0)) + Formula formula = null; + for (int i = 0; i < clauses.size(); i++) { + if (hit[i]) { + LogInfo.log("hit " + i); + continue; + } + // if (!hasHeadVar(clauses.get(i), headVar)) continue; + hit[i] = true; + LispTree tree = clauses.get(i); + Formula newFormula = toFormula(tree, headVar, existsVars); + if (formula == null) formula = newFormula; + else { + formula = new MergeFormula(MergeFormula.Mode.and, formula, newFormula); + } + } + return formula; + } + + Formula toOrFormula(List clauses, boolean[] hit, String headVar, List existsVars) { + // (or (town:t $1) (city:t $1)) + LogInfo.log("# or clauses: " + clauses.size()); + Formula formula = null; + for (int i = 0; i < clauses.size(); i++) { + if (hit[i]) { + LogInfo.log("hit " + i); + continue; + } + // if (!hasHeadVar(clauses.get(i), headVar)) continue; + hit[i] = true; + LispTree tree = clauses.get(i); + Formula newFormula = toFormula(tree, headVar, existsVars); + if (formula == null) formula = newFormula; + else { + formula = new MergeFormula(MergeFormula.Mode.or, formula, newFormula); + } + } + return formula; + } + + boolean validVar(String var, String headVar, List existsVars) { + if (var == null) + return false; + if (!var.startsWith("$")) + return true; + + boolean valid = false; + if (var.equals(headVar)) + valid = true; + if (existsVars != null && existsVars.contains(var)) + valid = true; + return valid; + } + + Formula toPrimitive(String s) { + String[] parts = s.split(":", 2); + if (parts.length < 2) + return null; + String type = parts[1]; + String primitive = primitiveMap.get(type); + if (primitive == null) + return null; + // if (primitive.equals("string")) + // parts[0] = capitalizeWords(parts[0]); + // return Formula.fromString(String.format("(%s %s)", primitive, parts[0])); + if (primitive.equals("string")) + return Formula.fromString(String.format("(!fb:type.object.name fb:en.%s)", parts[0])); + + return Formula.fromString(String.format("(%s %s)", primitive, parts[0])); + } + + Formula handleExists(LispTree tree, String headVar, List existsVars) { + // (exists $1 (and (river:t $1) (loc:t $1 $0))) + String eVar = tree.child(1).value; + existsVars.add(eVar); + // FIXME Currently assuming format where exists contains and statement with + // n clauses where first n - 1 clauses describe exists var and last clause + // relates exists var to the head var + List andTreeList = tree.child(2).children; + // FIXME Don't assume and tree + LispTree newAndTree = LispTree.proto.newList(); + + int predTreeInd = -1; + int numPredTrees = 0; + for (int k = 0; k < andTreeList.size(); k++) { + String childStr = andTreeList.get(k).toString(); + if (childStr.contains(headVar)) { + predTreeInd = k; + numPredTrees++; + } + } + + String func = tree.child(2).child(0).value; + if (numPredTrees == 0 || func.equals("exists")) { + return toLambda(eVar, toFormula(tree.child(2), eVar, existsVars)); + } + + newAndTree.children = new ArrayList(andTreeList.subList(0, andTreeList.size())); + LispTree predTree = newAndTree.children.remove(predTreeInd); + if (newAndTree.children.size() == 2) + newAndTree = newAndTree.child(1); + + LispTree newPredTree = LispTree.proto.parseFromString(predTree.toString().replace(eVar, newAndTree.toString())); + + Formula form = toFormula(newPredTree, eVar, existsVars); + existsVars.remove(eVar); + return form; + } + + // hit: ignore these clauses in (and ...) constructions + public Formula toFormula(LispTree tree, String headVar, List existsVars) { + + if (tree.isLeaf()) { + Formula s = toPrimitive(tree.toString()); + if (s != null) + return s; + return toPredicate(tree.toString()); + } + + String func = tree.child(0).value; + + if (func.equals("lambda")) { + // (lambda $0 e (and (river:t $0) (loc:t $0 arkansas:s))) + return toFormula(tree.child(3), tree.child(1).value, existsVars); + } else if (func.equals("count")) { + // (count $0 (and (river:t $0) (loc:t $0 washington:s))) + String countVar = tree.child(1).value; + if (headVar == null) + headVar = countVar; + if (!countVar.equals(headVar) && !existsVars.contains(countVar)) { + + return toLambda(countVar, new AggregateFormula(AggregateFormula.Mode.count, toFormula(tree.child(2), countVar, existsVars))); + } else + return new AggregateFormula(AggregateFormula.Mode.count, toFormula(tree.child(2), countVar, existsVars)); + } else if (func.equals("sum")) { + // (sum $0 (and (state:t $0) (next_to:t $0 texas:s)) (population:i $0)) + String numFunc = tree.child(3).child(0).value; + return new AggregateFormula(AggregateFormula.Mode.sum, + toJoin(numFunc, toFormula(tree.child(2), tree.child(1).value, existsVars))); + } else if (func.equals("argmax") || func.equals("argmin")) { + // (argmax $0 (state:t $0) (density:i $0)) + // (argmin $1 (river:t $1) (len:i $1)) + SuperlativeFormula.Mode mode = SuperlativeFormula.Mode.argmax; + if (func.equals("argmin")) + mode = SuperlativeFormula.Mode.argmin; + String superVar = tree.child(1).value; + Formula headFormula = toFormula(tree.child(2), superVar, existsVars); + Formula degreeFormula = toFormula(tree.child(3), superVar, existsVars); + NumberValue one = new NumberValue(1.0); + Formula rankFormula = new ValueFormula(one); + Formula countFormula = new ValueFormula(one); + // FIXME Hack to handle "most"/"least" expressions + if (tree.child(3).child(0).value.equals("count")) + degreeFormula = new ReverseFormula(degreeFormula); + // FIXME Currently assumes 1 1 + return new SuperlativeFormula(mode, rankFormula, countFormula, headFormula, degreeFormula); + } else if (func.equals("exists")) { + if (headVar == null) + headVar = tree.child(1).value; + return handleExists(tree, headVar, existsVars); + } else if (func.equals("not")) { + return new NotFormula(toFormula(tree.child(1), headVar, existsVars)); + } else if (func.equals("and")) { + return toAndFormula(tree.children.subList(1, tree.children.size()), new boolean[tree.children.size() - 1], headVar, existsVars); + } else if (func.equals("or")) { + return toOrFormula(tree.children.subList(1, tree.children.size()), new boolean[tree.children.size() - 1], headVar, existsVars); + } else { // Predicate + // Find head var + if (tree.children.size() == 2) { // Unary + if (isVar(tree.child(1).value)) { + // FIXME HACK + if (!getGeoType(func).equals("t")) + return toPredicate("!" + func); + return toPredicate(func); + } else + return toJoin(func, toFormula(tree.child(1), headVar, existsVars)); + } else if (tree.children.size() == 3) { // Binary + // FIXME Move elsewhere, there's both a binary and unary "capital:t" + if (func.equals("capital:t")) + func = "is_capital:t"; + + Formula form1 = toFormula(tree.child(1), headVar, existsVars); + Formula form2 = toFormula(tree.child(2), headVar, existsVars); + String first = tree.child(1).value; + String second = tree.child(2).value; + + // FIXME So ugly + boolean secondIsHead = isVar(second) && !existsVars.contains(second); + boolean firstIsHead = isVar(first) && (!existsVars.contains(first) || !secondIsHead); + if (firstIsHead) + return toJoin(func, form2); + else // secondIsHead + return toJoin("!" + func, form1); + // else + // return toJoin(func, form1); + } else { + throw new RuntimeException("Bad arity: " + tree); + } + } + } + + void countTokens(LispTree tree, Map counts) { + if (tree.isLeaf()) + MapUtils.incr(counts, tree.value, 1); + else { + for (LispTree child : tree.children) + countTokens(child, counts); + } + } + + String preprocessPredicates(String lispLine, String utterance) { + // Replacements + boolean replaced = false; + if (replaceMap.containsKey(lispLine)) { + lispLine = replaceMap.get(lispLine); + replaced = true; + } + + // FIXME Assumes that only one "major" in the line + // FIXME Specific to geoquery + if (utterance.contains("major river")) + lispLine = lispLine.replace("major", "major_river"); + else if (utterance.contains("major lake")) + lispLine = lispLine.replace("major", "major_lake"); + else + lispLine = lispLine.replace("major", "major_city"); + + if (utterance.contains("river")) + if (!replaced) + lispLine = lispLine.replace("loc:t", "river_loc:t"); + + lispLine = lispLine.replace("new_york:s", "new_york_state:s"); + + return lispLine; + } + + void convertExamples() { + String line; + int runInd = opts.runInd; + + try { + Example.Builder ex = null; + BufferedReader in = IOUtils.openIn(opts.inPath); + boolean gotUtterance = false; + int newId = 0; + String utterance = ""; + + while ((line = in.readLine()) != null) { + if (line.equals("") || line.startsWith("//")) continue; + if (ex == null) { + newId++; + ex = new Example.Builder(); + ex.setId("" + newId); + gotUtterance = false; + } + + + if (!gotUtterance) { + ex.setUtterance(line); + utterance = line; + gotUtterance = true; + } else { + if (runInd > 0 && newId != runInd) { + ex = null; + continue; + } + + LispTree tree = LispTree.proto.parseFromString(preprocessPredicates(line, utterance)); + LogInfo.logs(color.colorize("IN [%d]: %s", "blue"), newId, utterance); + LogInfo.logs(color.colorize("IN [%d]: %s", "purple"), newId, tree); + ArrayList existsVars = new ArrayList(); + + if (manualConversionsMap.containsKey(line)) { + ex.setTargetFormula(Formula.fromString(manualConversionsMap.get(line))); + } else { + ex.setTargetFormula(toFormula(tree, null, existsVars)); + } + LogInfo.logs(color.colorize("OUT [%d]: %s", "yellow"), newId, ex.createExample().targetFormula.toLispTree()); + examples.add(ex.createExample()); + ex = null; + } + } + in.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + void executeExamples() { + int runInd = opts.runInd; + // FIXME Should be passed in + SparqlExecutor.opts.endpointUrl = "http://localhost:3093/sparql"; + SparqlExecutor.opts.cachePath = "SparqlExecutor.cache"; + SparqlExecutor executor = new SparqlExecutor(); + + int exInd = 1; + if (runInd > 0) + exInd = runInd; + for (Example ex : examples) { + // Useful for just testing specific portion of examples + if (runInd > 0 && exInd != runInd) { + break; + } + + LogInfo.logs(color.colorize("[%d] %s", "blue"), exInd, ex.utterance); + LogInfo.logs(color.colorize("[%d] %s", "yellow"), exInd, ex.targetFormula.toString()); + try { + Executor.Response response = executor.execute(ex.targetFormula, null); + LogInfo.logs("\t\t [%d] %s", exInd, response.value.toString()); + validExampleIds.add(exInd); + validExampleLengths.add(ex.utterance.length()); + } catch (RuntimeException e) { + LogInfo.error(e); + failedExampleIds.add(exInd); + } + exInd++; + } + } + + void writeExamples() { + // Sort validExampleIds by the length of the utterance of the example + Collections.sort(validExampleIds, new Comparator() { + public int compare(Integer left, Integer right) { + return Integer.compare(validExampleLengths.get(validExampleIds.indexOf(left)), + validExampleLengths.get(validExampleIds.indexOf(right))); + } + }); + + PrintWriter out = IOUtils.openOutHard(opts.outPath); + out.println("["); // Print out as a list + String indent = " "; + for (int k = 0; k < validExampleIds.size(); k++) { + int j = validExampleIds.get(k) - 1; + Example ex = examples.get(j); + if (k < validExampleIds.size() - 1) + out.println(indent + ex.toJson() + ","); + else + out.println(indent + ex.toJson()); + } + out.println("]"); // Print out as a list + out.close(); + } + + void printSummary() { + LogInfo.logs("%d input examples", examples.size()); + LogInfo.logs("%d successful executions", validExampleIds.size()); + LogInfo.logs("Failed executions (%d):", failedExampleIds.size()); + for (int k : failedExampleIds) { + Example ex = examples.get(k - 1); + LogInfo.log(ex.toJson()); + } + } + + public static void main(String[] args) throws InterruptedException { + Execution.run(args, new LambdaCalculusConverter(), "lcc", LambdaCalculusConverter.opts); + } +} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/Lexicon.java b/src/edu/stanford/nlp/sempre/freebase/Lexicon.java similarity index 67% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/Lexicon.java rename to src/edu/stanford/nlp/sempre/freebase/Lexicon.java index 6d942bb..8967274 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/Lexicon.java +++ b/src/edu/stanford/nlp/sempre/freebase/Lexicon.java @@ -1,8 +1,9 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; +package edu.stanford.nlp.sempre.freebase; -import edu.stanford.nlp.sempre.StringCache; -import edu.stanford.nlp.sempre.StringCacheUtils; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.LexicalEntrySerializer; +import edu.stanford.nlp.sempre.cache.StringCache; +import edu.stanford.nlp.sempre.cache.StringCacheUtils; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.LexicalEntrySerializer; import fig.basic.LispTree; import fig.basic.LogInfo; import fig.basic.Option; @@ -12,33 +13,38 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -public class Lexicon { +public final class Lexicon { public static class Options { - @Option(gloss = "The path for the cache") + @Option(gloss = "The path for the cache") public String cachePath; - @Option(gloss = "Search strategies for entities: exact, inexact, fbsearch") - public EntityLexicon.SearchStrategy entitySearchStrategy = EntityLexicon.SearchStrategy.inexact; } public static Options opts = new Options(); private static Lexicon lexicon; - public static Lexicon getSingleton() throws IOException { - if(lexicon==null) - lexicon=new Lexicon(); - return lexicon; + public static Lexicon getSingleton() { + try { + if (lexicon == null) + lexicon = new Lexicon(); + return lexicon; + } catch (IOException e) { + throw new RuntimeException(e); + } } public StringCache cache; private EntityLexicon entityLexicon; private UnaryLexicon unaryLexicon; - protected BinaryLexicon binaryLexicon; + private BinaryLexicon binaryLexicon; + + public EntityLexicon getEntityLexicon() { return entityLexicon; } private Lexicon() throws IOException { LogInfo.begin_track("Lexicon()"); - entityLexicon = new EntityLexicon(); - unaryLexicon = new UnaryLexicon(); - binaryLexicon = new BinaryLexicon(); + // TODO(joberant): why is BinaryLexicon special? -- wait why is it special? + entityLexicon = EntityLexicon.getInstance(); + unaryLexicon = UnaryLexicon.getInstance(); + binaryLexicon = BinaryLexicon.getInstance(); LogInfo.end_track(); if (opts.cachePath != null) @@ -65,11 +71,11 @@ public class Lexicon { String key = mode + ":" + query; String response; synchronized (cache) { - response = cache.get(key); + response = cache.get(key); } if (response == null) return null; LispTree tree = LispTree.proto.parseFromString(response); - List entries = new ArrayList(); + List entries = new ArrayList<>(); for (int i = 0; i < tree.children.size(); i++) entries.add(LexicalEntrySerializer.entryFromLispTree(tree.child(i))); return entries; @@ -82,7 +88,7 @@ public class Lexicon { for (LexicalEntry entry : entries) result.addChild(LexicalEntrySerializer.entryToLispTree(entry)); synchronized (cache) { - cache.put(key, result.toString()); - } + cache.put(key, result.toString()); + } } } diff --git a/src/edu/stanford/nlp/sempre/freebase/LexiconFn.java b/src/edu/stanford/nlp/sempre/freebase/LexiconFn.java new file mode 100644 index 0000000..e702376 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/LexiconFn.java @@ -0,0 +1,412 @@ +package edu.stanford.nlp.sempre.freebase; + +import com.google.common.base.Joiner; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.BinaryLexicalEntry; +import fig.basic.*; +import org.apache.lucene.queryparser.classic.ParseException; + +import java.io.IOException; +import java.util.*; + +/** + * Looks up a string into the lexicon, interfacing with the fbalignment code. + * @author Percy Liang + */ +public class LexiconFn extends SemanticFn { + public static class Options { + // Note: we filter here and not in entity lexicon so that don't need different cache for different numbers. + @Option(gloss = "Number of entities to return from entity lexicon") + public int maxEntityEntries = 100; + @Option public int maxUnaryEntries = Integer.MAX_VALUE; + @Option public int maxBinaryEntries = Integer.MAX_VALUE; + + @Option(gloss = "Verbose") public int verbose = 0; + @Option(gloss = "Class name for lexicon") public String lexiconClassName; + @Option public boolean useHistogramFeatures = true; + } + + public static Options opts = new Options(); + private static Lexicon lexicon; + public static Evaluation lexEval = new Evaluation(); + + private String mode; // unary, binary, or entity + private EntityLexicon.SearchStrategy entitySearchStrategy; // For entities, how to search + private TextToTextMatcher textToTextMatcher = new TextToTextMatcher(); + private FbFormulasInfo fbFormulaInfo; + + public static final Map, Set> correctEntryToExampleIds = + new HashMap<>(); + + public LexiconFn() throws IOException { + lexicon = Lexicon.getSingleton(); + fbFormulaInfo = FbFormulasInfo.getSingleton(); + } + + public void init(LispTree tree) { + super.init(tree); + for (int i = 1; i < tree.children.size(); i++) { + String value = tree.child(i).value; + + // mode + if (value.equals("unary")) this.mode = "unary"; + else if (value.equals("binary")) this.mode = "binary"; + else if (value.equals("entity")) this.mode = "entity"; + // entity search strategy + // TODO(joberant): aren't we saying these should be set by a flag so the cache doesn't get messed up? + else if (value.equals("inexact")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.inexact; + else if (value.equals("exact")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.exact; + else if (value.equals("fbsearch")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.fbsearch; + else throw new RuntimeException("Invalid argument: " + value); + } + } + + public static void getEntityEntryFeatures(LexicalEntry.EntityLexicalEntry eEntry, FeatureVector features, + Callable c, Example ex) { + + if (FeatureExtractor.containsDomain("basicStats")) { + if (opts.useHistogramFeatures) + features.addHistogram("basicStats", "entity.popularity ", eEntry.getPopularity()); + else + features.addWithBias("basicStats", "entity.popularity", Math.log(eEntry.getPopularity() + 1)); + } + if (FeatureExtractor.containsDomain("entityFeatures")) { + for (String feature : eEntry.entityFeatures.keySet()) { + double value = eEntry.entityFeatures.getCount(feature); + features.addWithBias("entityFeatures", "entity." + feature, value); + } + features.add("entityFeatures", "entity.pos=" + ex.languageInfo.posSeq(c.getStart(), c.getEnd())); + features.add("entityFeatures", "entity.mention_length=" + (c.getEnd() - c.getStart())); + } + if (FeatureExtractor.containsDomain("lexAlign")) + features.add("lexAlign", eEntry.textDescription + " --- " + eEntry.formula); + } + + public static void getUnaryEntryFeatures(LexicalEntry.UnaryLexicalEntry uEntry, FeatureVector features) { + if (FeatureExtractor.containsDomain("basicStats")) { + if (opts.useHistogramFeatures) + features.addHistogram("basicStats", "unary.popularity ", uEntry.getPopularity() + 1); + else + features.addWithBias("basicStats", "unary.popularity", Math.log(uEntry.getPopularity() + 1)); + } + // Alignment scores features + if (FeatureExtractor.containsDomain("alignmentScores")) { + for (String feature : uEntry.alignmentScores.keySet()) { + features.addWithBias("alignmentScores", "unary." + feature, Math.log(MapUtils.getDouble(uEntry.alignmentScores, feature, 0.0) + 1)); + } + } + + if (FeatureExtractor.containsDomain("basicStats")) { + if (uEntry.getDistance() < 0.0001) + features.add("basicStats", "unary.equal"); + + // adding the source of the lexical entry as a feature + features.add("basicStats", "unary.source=" + uEntry.source); + } + if (FeatureExtractor.containsDomain("lexAlign")) + features.add("lexAlign", uEntry.textDescription + " --- " + uEntry.formula); + } + + public static void getBinaryEntryFeatures(BinaryLexicalEntry bEntry, FeatureVector features) { + if (FeatureExtractor.containsDomain("basicStats")) { + if (opts.useHistogramFeatures) + features.addHistogram("basicStats", "binary.popularity ", bEntry.getPopularity() + 1); + else + features.addWithBias("basicStats", "binary.popularity", Math.log(bEntry.getPopularity() + 1)); + // adding the source of the lexical entry as a feature + features.add("basicStats", "binary." + bEntry.source); + } + // Alignment scores features + if (FeatureExtractor.containsDomain("alignmentScores")) { + for (String feature : bEntry.alignmentScores.keySet()) { + features.addWithBias("alignmentScores", "binary." + feature, Math.log(MapUtils.getDouble(bEntry.alignmentScores, feature, 0.0) + 1)); + } + } + if (FeatureExtractor.containsDomain("lexAlign")) + features.add("lexAlign", bEntry.textDescription + " --- " + bEntry.formula); + } + + // Convert LexicalEntry into a form consumable by the semantic parser. + private Derivation convert(Example ex, + Callable c, + String mode, + String word, + LexicalEntry entry) { + FeatureVector features = new FeatureVector(); + SemType type; + + switch (mode) { + case "entity": + // Entities + LexicalEntry.EntityLexicalEntry eEntry = (LexicalEntry.EntityLexicalEntry) entry; + getEntityEntryFeatures(eEntry, features, c, ex); + type = ((LexicalEntry.EntityLexicalEntry) entry).type; + break; + case "unary": + // Unaries + LexicalEntry.UnaryLexicalEntry uEntry = (LexicalEntry.UnaryLexicalEntry) entry; + getUnaryEntryFeatures(uEntry, features); + type = ((LexicalEntry.UnaryLexicalEntry) entry).type; + break; + case "binary": + // Binaries + BinaryLexicalEntry bEntry = (BinaryLexicalEntry) entry; + getBinaryEntryFeatures(bEntry, features); + // features that depend on entry but also on the example + features.add( + textToTextMatcher.extractFeatures( + ex.languageInfo.tokens.subList(c.getStart(), c.getEnd()), + ex.languageInfo.posTags.subList(c.getStart(), c.getEnd()), + ex.languageInfo.lemmaTokens.subList(c.getStart(), c.getEnd()), + bEntry.fbDescriptions)); + + // Note that expectedType2 is the argument type, expectedType1 is the return type. + type = SemType.newFuncSemType(bEntry.getExpectedType2(), bEntry.getExpectedType1()); + break; + default: + throw new RuntimeException("Invalid mode: " + mode); + } + + Derivation newDeriv = new Derivation.Builder() + .withCallable(c) + .formula(entry.formula) + .type(type) + .localFeatureVector(features) + .createDerivation(); + + if (SemanticFn.opts.trackLocalChoices) + newDeriv.addLocalChoice("LexiconFn " + newDeriv.startEndString(ex.getTokens()) + " " + entry); + + if (opts.verbose >= 3) { + LogInfo.logs( + "LexiconFn: %s [%s => %s ~ %s | %s]: popularity = %s, distance = %s, type = %s, source=%s", + mode, word, entry.normalizedTextDesc, entry.fbDescriptions, newDeriv.formula, + entry.getPopularity(), entry.getDistance(), newDeriv.type, entry.source); + } + return newDeriv; + } + + public DerivationStream call(Example ex, Callable c) { + + if (opts.verbose >= 5) LogInfo.begin_track("LexicalFn.call: %s", c.childStringValue(0)); + + String query = c.childStringValue(0); + DerivationStream res; + + try { + switch (mode) { + // Entities + case "entity": { + // if (opts.verbose >= 2) + // LogInfo.log("LexiconFn: querying for entity: " + query); + + List entries = lexicon.lookupEntities(query, entitySearchStrategy); + lexEval.add("entity", !entries.isEmpty()); + entries = entries.subList(0, Math.min(opts.maxEntityEntries, entries.size())); + res = new LazyLexiconFnDerivs(ex, c, entries, query); + break; + } + // Unaries + case "unary": { + List entries = lexicon.lookupUnaryPredicates(query); + lexEval.add("unary", !entries.isEmpty()); + entries = entries.subList(0, Math.min(opts.maxUnaryEntries, entries.size())); + res = new LazyLexiconFnDerivs(ex, c, entries, query); + break; + } + // Binaries + case "binary": { + List entries = lexicon.lookupBinaryPredicates(query); + lexEval.add("binary", !entries.isEmpty()); + List filteredEntries = new ArrayList<>(); + // filter cvt entries (TODO(joberant): remove this hack) + for (LexicalEntry entry : entries) { + if (!fbFormulaInfo.isCvt(((BinaryLexicalEntry) entry).expectedType1) + && !fbFormulaInfo.isCvt(((BinaryLexicalEntry) entry).expectedType2)) + filteredEntries.add(entry); + } + filteredEntries = filteredEntries.subList(0, Math.min(opts.maxBinaryEntries, filteredEntries.size())); + res = new LazyLexiconFnDerivs(ex, c, filteredEntries, query); + break; + } + default: + throw new RuntimeException("Illegal mode: " + mode); + } + } catch (IOException | ParseException e) { + throw new RuntimeException(e); + } + + if (opts.verbose >= 5) LogInfo.end_track(); + return res; + } + + // if there was bridging then have a rule from tokens to binary + @Override + public void addFeedback(Example ex) { + LogInfo.begin_track("LexiconFn.addFeedback"); + Set> correctLexemeFormulaMatches = collectLexemeFormulaPairs(ex); + + for (Pair pair: correctLexemeFormulaMatches) { + LogInfo.logs("LexiconFn.addFeedback: %s => %s", pair.getFirst(), pair.getSecond()); + // TODO(joberant): hack to get id + MapUtils.addToSet(correctEntryToExampleIds, pair, Integer.parseInt(ex.id.substring(ex.id.lastIndexOf(':') + 1))); + BinaryLexicon.getInstance().updateLexicon(pair, correctEntryToExampleIds.get(pair).size()); + } + LogInfo.end_track(); + } + + private static Set> collectLexemeFormulaPairs(Example ex) { + Set> res = new HashSet<>(); + Set> temp = new HashSet<>(); + for (Derivation correctDerivation : ex.getCorrectDerivations()) { + // get all join formulas + List relations = + correctDerivation.formula.mapToList(formula -> { + List res1 = new ArrayList<>(); + if (formula instanceof JoinFormula) + res1.add(((JoinFormula) formula).relation); + return res1; + }, false); + Set validIndices = new HashSet<>(); + findValidIndices(correctDerivation, validIndices); + // match formulas + for (Formula relation : relations) { + for (int i = 0; i < ex.numTokens(); ++i) { + if (LanguageInfo.isContentWord(ex.posTag(i)) && + validIndices.contains(i)) { + temp.add(Pair.newPair(ex.languageInfo.lemmaTokens.get(i), relation)); + } + } + } + } + // reverse invalid relations + for (Pair pair: temp) { + if (!BinaryLexicon.getInstance().validBinaryFormula(pair.getSecond())) { + pair.setSecond(FbFormulasInfo.getSingleton().equivalentFormula(pair.getSecond())); + } + res.add(pair); + } + return res; + } + + @Override + public void sortOnFeedback(Params params) { + LogInfo.begin_track("Learner.sortLexiconOnFeedback"); + BinaryLexicon.getInstance().sortLexiconByFeedback(params); + UnaryLexicon.getInstance().sortLexiconByFeedback(params); + LogInfo.end_track(); + } + + // todo - this method is grammar specific and that is bad + private static void findValidIndices(Derivation deriv, Set indices) { + if (deriv.cat.equals("$Entity")) + return; + if (deriv.children.size() == 0) { + for (int i = deriv.start; i < deriv.end; ++i) + indices.add(i); + return; + } + for (Derivation child : deriv.children) + findValidIndices(child, indices); + } + + /** For now this ignores stemming!!! */ + private static boolean doesContextMatch(Example ex, Derivation deriv, BinaryLexicalEntry bEntry) { + + if (bEntry.isFullLexemeEqualToNormalizedText()) + return true; + // get the left and right context surrounding the core (normalized text) + String[] leftContext = bEntry.getLeftContext(); + String[] rightContext = bEntry.getRightContext(); + // match right context + for (int i = 0; i < rightContext.length; ++i) { + // in this case all context words were matched and some were dropped but there was no mismatch + if (deriv.end + i >= ex.numTokens() || ex.token(deriv.end + i).equals("?")) + break; + + if (!rightContext[0].equals(ex.lemmaToken(deriv.end + i))) { + if (opts.verbose >= 4) { + LogInfo.logs( + "RIGHT CONTEXT MISMATCH: full lexeme=%s, normalized text=%s left context=%s, right context=%s example=%s, formula=%s", + bEntry.fullLexeme, + bEntry.normalizedTextDesc, + Joiner.on(' ').join(leftContext), + Joiner.on(' ').join(rightContext), + Joiner.on(' ').join(ex.languageInfo.tokens), + bEntry.formula); + } + return false; + } + } + + // match right context + for (int i = 0; i < leftContext.length; ++i) { + if (deriv.start - i - 1 < 0) // in this case all context words were matched and some were dropped but there was no mismatch + break; + if (!leftContext[leftContext.length - i - 1].equals(ex.lemmaToken(deriv.start - i - 1))) { + if (opts.verbose >= 2) { + LogInfo.logs( + "LEFT CONTEXT MISMATCH: full lexeme=%s, normalized text=%s left context=%s, right context=%s example=%s, formula=%s", + bEntry.fullLexeme, + bEntry.normalizedTextDesc, + Joiner.on(' ').join(leftContext), + Joiner.on(' ').join(rightContext), + Joiner.on(' ').join(ex.languageInfo.tokens), + bEntry.formula); + } + return false; + } + } + return true; + } + + public class LazyLexiconFnDerivs extends MultipleDerivationStream { + + private Example ex; + private Callable callable; + private List entries; // we get one derivation from each entry + private String query; + private int currIndex = 0; + + public LazyLexiconFnDerivs(Example ex, Callable c, List entries, String query) { + this.ex = ex; + this.callable = c; + this.entries = entries; + this.query = query; + } + + @Override + public int estimatedSize() { + return entries.size() - currIndex; + } + + @Override + public Derivation createDerivation() { + if (currIndex == entries.size()) + return null; + LexicalEntry currEntry = entries.get(currIndex++); + Derivation res; + switch (mode) { + case "entity": + res = convert(ex, callable, "entity", query, currEntry); + break; + case "unary": + res = convert(ex, callable, "unary", query, currEntry); + break; + case "binary": + res = convert(ex, callable, "binary", query, currEntry); + // add context matching feature + if (FeatureExtractor.containsDomain("context")) { + if (!doesContextMatch(ex, res, (BinaryLexicalEntry) currEntry)) + res.addFeature("context", "binary.contextMismatch"); + } + break; + default: + throw new RuntimeException("Illegal mode: " + mode); + } + return res; + } + } +} + diff --git a/src/edu/stanford/nlp/sempre/SparqlExecutor.java b/src/edu/stanford/nlp/sempre/freebase/SparqlExecutor.java similarity index 51% rename from src/edu/stanford/nlp/sempre/SparqlExecutor.java rename to src/edu/stanford/nlp/sempre/freebase/SparqlExecutor.java index 60f4ea0..f6d0aa7 100644 --- a/src/edu/stanford/nlp/sempre/SparqlExecutor.java +++ b/src/edu/stanford/nlp/sempre/freebase/SparqlExecutor.java @@ -1,5 +1,8 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.freebase; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.cache.StringCache; +import edu.stanford.nlp.sempre.cache.StringCacheUtils; import fig.basic.*; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -48,16 +51,19 @@ public class SparqlExecutor extends Executor { @Option(gloss = "URL where the SPARQL server lives") public String endpointUrl; - @Option(gloss = "Whether to return a table of results rather than just a list of results") + @Option(gloss = "Whether to return a table of results rather than a list of entities (needed to support 'capital of each state')") public boolean returnTable = false; - // TODO: remove this since this is a really bad hack. + // FIXME TODO(pliang): remove this since this is a really bad hack. @Option(gloss = "If false, then enforce that denotation of (lambda x (border x)) does not contain (x,x)") - public boolean lambdaAllowDiagonals = false; + public boolean lambdaAllowDiagonals = true; @Option(gloss = "Whether to include entity names (mostly for readability)") public boolean includeEntityNames = true; + @Option(gloss = "Whether to return supporting information (e.g., 'length' for the 'longest river')") + public boolean includeSupportingInfo = false; + @Option public int verbose = 1; } @@ -155,7 +161,7 @@ public class SparqlExecutor extends Executor { // - XML // - Whether to print out details (coincides with whether this query was cached). public ServerResponse runQueryToGetXml(String queryStr, Formula formula) { - //LogInfo.logs("%s %s", formula, queryStr); + if (opts.verbose >= 3) LogInfo.logs("SparqlExecutor.execute: %s", queryStr); ServerResponse response = null; // Note: only cache for concrete queries. @@ -176,7 +182,7 @@ public class SparqlExecutor extends Executor { } // If not cached, then make the actual request. - //if (response == null || response.xml == null || response.xml.contains("TIMEOUT")) { + // if (response == null || response.xml == null || response.xml.contains("TIMEOUT")) { if (response == null) { // Note: begin_track without end_track if (opts.verbose >= 1) { @@ -212,7 +218,7 @@ public class SparqlExecutor extends Executor { throw new RuntimeException(e); } catch (SAXException e) { LogInfo.errors("XML: %s", xml); - //throw new RuntimeException(e); + // throw new RuntimeException(e); return null; } catch (ParserConfigurationException e) { throw new RuntimeException(e); @@ -221,10 +227,14 @@ public class SparqlExecutor extends Executor { } // Main entry point. - public Response execute(Formula formula) { + public Response execute(Formula formula, ContextValue context) { + // Note: don't do beta reduction here to preserve the semantics of lambda DCS. + // Beta reduction should be triggered deliberately in the SemanticFn. return execute(formula, 0, opts.maxResults); } public synchronized Response execute(Formula formula, int offset, int maxResults) { + if (opts.verbose >= 3) + LogInfo.logs("SparqlExecutor.execute: %s", formula); String prefix = "exec-"; Evaluation stats = new Evaluation(); @@ -272,11 +282,11 @@ public class SparqlExecutor extends Executor { // Extract the results from XML now. NodeList results = extractResultsFromXml(serverResponse.xml); if (results == null) return new Response(ErrorValue.badFormat, stats); - List values = new ValuesExtractor(serverResponse.beginTrack, formula, converter).extract(results); + Value value = new ValuesExtractor(serverResponse.beginTrack, formula, converter).extract(results); if (serverResponse.beginTrack && opts.verbose >= 1) LogInfo.end_track(); - return new Response(new ListValue(values), stats); + return new Response(value, stats); } //////////////////////////////////////////////////////////// @@ -284,13 +294,46 @@ public class SparqlExecutor extends Executor { class Converter { private int numVars = 0; // Used to create new Sparql variables + // Unit for each SPARQL variable. + Map unitsMap = new HashMap(); // ?y => fb:en.meter + + // For each variable, a description + Map descriptionsMap = new HashMap(); // ?y => "Height (meters)" + String queryStr; SparqlSelect query; // Resulting SPARQL expression + // The state used to a SELECT statement (DRT box in which all variables are existentially closed). + class Box { + // These are the variables that are first selected. + List initialVars = new ArrayList(); + + // Mapping from lambda DCS variables to SPARQL variables (which are unique across the entire formula, not just this box). + Map env = new LinkedHashMap(); + + // Some SPARQL variables are bound to quantities based on the SELECT statement (e.g., COUNT(?x)). + Map asValuesMap = new HashMap(); // e.g., ?y => COUNT(?x) or ?y => (?x1 + ?x2) + } + public Converter(Formula rootFormula, int offset, int maxResults) throws BadFormulaException { Ref head = new Ref(); - Map env = new LinkedHashMap(); - query = closeExistentialScope(convert(rootFormula, head, null, env), head, env, null, null); + Box box = new Box(); + + rootFormula = stripOuterLambdas(box, rootFormula); + SparqlBlock block = convert(rootFormula, head, null, box); + query = closeExistentialScope(block, head, box); + + // Select all the variables that appear in the block if we want to return a table. + // (Provide the evidence.) + if (opts.includeSupportingInfo) { + for (SparqlExpr expr : block.children) { + if (!(expr instanceof SparqlStatement)) continue; + SparqlStatement stmt = (SparqlStatement) expr; + if (stmt.arg1 instanceof VariableFormula) addSelectVar(box, query, block, (VariableFormula) stmt.arg1, false); + if (stmt.arg2 instanceof VariableFormula) addSelectVar(box, query, block, (VariableFormula) stmt.arg2, false); + } + } + if (query.offset == 0) // If not set query.offset = offset; if (query.limit == -1) // If not set @@ -298,96 +341,179 @@ public class SparqlExecutor extends Executor { queryStr = "PREFIX fb: <" + FreebaseInfo.freebaseNamespace + "> " + query; } - // Create a SELECT expression. - // Note: corresponds to a box in DRT. - private SparqlSelect closeExistentialScope(SparqlBlock block, Ref head, - Map env, - String headAsValue, String headUnit) { + // Strip off lambdas and add the variables to the environment + // For example, in (lambda x (lambda y BODY)), we would create an + // environment {x:NEW_VAR, y:NEW_VAR}, and interpret BODY as a unary. + private Formula stripOuterLambdas(Box box, Formula formula) { + while (formula instanceof LambdaFormula) { + LambdaFormula lambda = ((LambdaFormula) formula); + VariableFormula var = newVar(); + box.env.put(new VariableFormula(lambda.var), var); + box.initialVars.add(var); + formula = lambda.body; + } + return formula; + } + + // Create a SELECT expression (a DRT box). + private SparqlSelect closeExistentialScope(SparqlBlock block, Ref head, Box box) { + // Optimization: if block only contains one select statement, then can optimize and just return that. + if (block.children.size() == 1 && block.children.get(0) instanceof SparqlSelect && box.initialVars.size() == 0) + return (SparqlSelect) block.children.get(0); + SparqlSelect select = new SparqlSelect(); - // Ensure that the head is a variable rather than a primitive value. - VariableFormula headVar = ensureIsVar(block, head); - if (headUnit == null) headUnit = getUnit(block, headVar); - select.selectVars.add(new SparqlSelect.Var(headVar, headAsValue, headUnit, false)); + // Add initial variables + for (VariableFormula var : box.initialVars) + addSelectVarWithName(box, select, block, var); - // Get the name of the head - if (opts.includeEntityNames && FreebaseInfo.ENTITY.equals(headUnit)) { - VariableFormula headNameVar = new VariableFormula(headVar.name + "name"); - select.selectVars.add(new SparqlSelect.Var(headNameVar, null, FreebaseInfo.TEXT, true)); - block.addOptionalStatement(headVar, FreebaseInfo.NAME, headNameVar); - } + // Add head variable (ensure that the head is a variable rather than a primitive value) + VariableFormula headVar = ensureIsVar(box, block, head); + addSelectVarWithName(box, select, block, headVar); - // Add the other variables in the environment (these are things we need to condition on) - for (PrimitiveFormula formula : env.values()) { + // Add the other supporting variables in the environment (for communicating with nested blocks, e.g. for superlatives). + for (PrimitiveFormula formula : box.env.values()) { if (!(formula instanceof VariableFormula)) continue; - VariableFormula condition = (VariableFormula)formula; - if (headVar.equals(condition)) continue; // Don't select the head again, since we already added that. - select.selectVars.add(new SparqlSelect.Var(condition, null, getUnit(block, condition), false)); + VariableFormula supportingVar = (VariableFormula) formula; + addSelectVarWithName(box, select, block, supportingVar); } select.where = block; return select; } + // Add the variable |var|, but also potentially + private void addSelectVarWithName(Box box, SparqlSelect select, SparqlBlock block, VariableFormula var) { + addSelectVar(box, select, block, var, false); + + // Get the name of the head + String unit = unitsMap.get(var); + // LogInfo.logs("unit[%s] = %s", var, unit); + if (opts.includeEntityNames && FreebaseInfo.ENTITY.equals(unit)) { + VariableFormula nameVar = new VariableFormula(var.name + "name"); + addSelectVar(box, select, block, nameVar, true); + addOptionalStatement(block, var, FreebaseInfo.NAME, nameVar); + } + } + + private void addSelectVar(Box box, SparqlSelect select, SparqlBlock block, VariableFormula var, boolean isAuxiliary) { + if (opts.verbose >= 5) LogInfo.logs("addSelectVar: %s : %s | %s", var, box.asValuesMap.get(var), box.asValuesMap); + + // Check if alrady exists; if so, don't add it again + for (SparqlSelect.Var oldVar : select.selectVars) + if (oldVar.var.equals(var)) return; + + select.selectVars.add(new SparqlSelect.Var(var, box.asValuesMap.get(var), unitsMap.get(var), isAuxiliary, descriptionsMap.get(var))); + } + // Mutable |head| to make sure it contains a VariableFormula. - private VariableFormula ensureIsVar(SparqlBlock block, Ref head) { + private VariableFormula ensureIsVar(Box box, SparqlBlock block, Ref head) { VariableFormula headVar; if (head.value instanceof VariableFormula) { - headVar = (VariableFormula)head.value; + headVar = (VariableFormula) head.value; } else { headVar = newVar(); - if (head.value != null) - block.addStatement(headVar, "=", head.value); + if (head.value != null) { + // LogInfo.logs("ensureIsVar: %s : %s", headVar, head.value); + Value value = ((ValueFormula) head.value).value; + if (value instanceof NumberValue) { // encode as (3 as ?x1) [FILTER doesn't work for isolated numbers] + box.asValuesMap.put(headVar, Formulas.getString(head.value)); + unitsMap.put(headVar, valueToUnit(((ValueFormula) head.value).value)); + } else { // encode as (FILTER (?x1 = fb:en.barack_obama)) + addStatement(block, headVar, "=", head.value); + addEntityStatement(block, headVar); + } + } head.value = headVar; } return headVar; } - // Return the unit (e.g., fb:en.meter) that |var| has in |expr|. - // NOTE: this function is incomplete currently and might not always be - // able to infer the unit; in that case, it will just return null. - private String getUnit(SparqlExpr expr, VariableFormula var) { - if (expr instanceof SparqlStatement) { - SparqlStatement statement = (SparqlStatement)expr; - // If the statement is ?x = , then try to figure out the unit of value. - if (var.equals(statement.arg1)) { - if (statement.relation.equals("=") && statement.arg2 instanceof ValueFormula) { - Value value = ((ValueFormula)statement.arg2).value; - if (value instanceof NameValue) return FreebaseInfo.ENTITY; - if (value instanceof BooleanValue) return FreebaseInfo.BOOLEAN; - if (value instanceof NumberValue) return ((NumberValue)value).unit; - if (value instanceof StringValue) return FreebaseInfo.TEXT; - if (value instanceof DateValue) return FreebaseInfo.DATE; - } - if (!SparqlStatement.isOperator(statement.relation)) - return FreebaseInfo.ENTITY; + // Add statement as well as updating the units information. + private void addOptionalStatement(SparqlBlock block, PrimitiveFormula arg1, String property, PrimitiveFormula arg2) { addStatement(block, arg1, property, arg2, true); } + private void addStatement(SparqlBlock block, PrimitiveFormula arg1, String property, PrimitiveFormula arg2) { addStatement(block, arg1, property, arg2, false); } + private void addStatement(SparqlBlock block, PrimitiveFormula arg1, String property, PrimitiveFormula arg2, boolean optional) { + block.addStatement(arg1, property, arg2, optional); + + if (arg1 instanceof VariableFormula) { + VariableFormula var = (VariableFormula) arg1; + + // If the statement is ?x = , then extract unit from value. + if (property.equals("=") && arg2 instanceof ValueFormula) { + updateUnit(var, valueToUnit(((ValueFormula) arg2).value)); + } else if (property.equals(FreebaseInfo.TYPE)) { + String type = Formulas.getString(arg2); + updateUnit(var, fbInfo.typeToUnit(type, null)); + if (descriptionsMap.get(var) == null) + descriptionsMap.put(var, fbInfo.getName(type)); + } else if (!SparqlStatement.isOperator(property)) { + updateUnit(var, fbInfo.getUnit1(property)); } - if (var.equals(statement.arg2)) - return fbInfo.getUnit2(statement.relation); - } else if (expr instanceof SparqlBlock) { - for (SparqlExpr subexpr : ((SparqlBlock)expr).children) { - String unit = getUnit(subexpr, var); - if (unit != null) return unit; + + if (descriptionsMap.get(var) == null) { + descriptionsMap.put(var, fbInfo.getName(fbInfo.getArg1Type(property))); + if (opts.verbose >= 3) LogInfo.logs("description arg1=%s => %s => %s", var, fbInfo.getArg1Type(property), descriptionsMap.get(var)); } - } else if (expr instanceof SparqlSelect) { - SparqlSelect select = (SparqlSelect)expr; - for (SparqlSelect.Var selectVar : select.selectVars) { - if (selectVar.var.equals(var)) return selectVar.unit; - } - return getUnit(select.where, var); } + + if (arg2 instanceof VariableFormula) { + // Get unit from Freebase property. + VariableFormula var = (VariableFormula) arg2; + updateUnit(var, fbInfo.getUnit2(property)); + descriptionsMap.put(var, fbInfo.getName(property)); + if (opts.verbose >= 3) LogInfo.logs("description arg2=%s => %s => %s", var, property, descriptionsMap.get(var)); + } + } + + void addEntityStatement(SparqlBlock block, VariableFormula var) { + // This is dangerous because in the DB, not all entities are necessarily labeled with fb:common.topic + //addStatement(block, var, FreebaseInfo.TYPE, new ValueFormula(new NameValue(FreebaseInfo.ENTITY))); + // Only needed when includeEntityNames = true. + addStatement(block, var, FreebaseInfo.TYPE, newVar()); + } + + // Update the unit of |var| if necessary. + void updateUnit(VariableFormula var, String unit) { + if (opts.verbose >= 5) LogInfo.logs("updateUnit: %s : %s", var, unit); + if (unit == null) return; + String oldUnit = unitsMap.get(var); + if (oldUnit == null) { unitsMap.put(var, unit); return; } + + // This replacement isn't quite kosher from a subtyping relation point of + // view (we're dealing with units, not types). + if (oldUnit.equals(NumberValue.unitless)) { unitsMap.put(var, unit); return; } // fb:en.meter replaces fb:en.unitless + if (oldUnit.equals(FreebaseInfo.ENTITY)) { unitsMap.put(var, unit); return; } // fb:en.cvt replaces fb:common.topic + if (oldUnit.equals(FreebaseInfo.CVT)) return; // Keep CVT + + if (!unit.equals(oldUnit)) + LogInfo.errors("Unit mis-match for %s: old is '%s', new is '%s'", var, oldUnit, unit); + } + + void updateAsValues(Box box, VariableFormula var, String asValue) { + if (opts.verbose >= 5) LogInfo.logs("updateAsValues: %s : %s", var, asValue); + box.asValuesMap.put(var, asValue); + } + + private String valueToUnit(Value value) { + // Note: units are fine grained on numbers but coarse on entities. + if (value instanceof NameValue) return FreebaseInfo.ENTITY; // Assume this is not a CVT + if (value instanceof BooleanValue) return FreebaseInfo.BOOLEAN; + if (value instanceof NumberValue) return ((NumberValue) value).unit; + if (value instanceof StringValue) return FreebaseInfo.TEXT; + if (value instanceof DateValue) return FreebaseInfo.DATE; return null; } + // Main conversion function. // head, modifier: SPARQL variables (e.g., ?x13) - // env: mapping from lambda-DCS variables (e.g., ?city) to SPARQL variables (?x13) - private SparqlBlock convert(Formula rawFormula, - Ref head, Ref modifier, - Map env) { - if (opts.verbose >= 5) LogInfo.begin_track("convert %s: head = %s, modifier = %s, env = %s", rawFormula, head, modifier, env); + // box: + // - env: mapping from lambda-DCS variables (e.g., ?city) to SPARQL variables (?x13) + // - asValuesMap: additional constraints + private SparqlBlock convert(Formula rawFormula, Ref head, Ref modifier, Box box) { + if (opts.verbose >= 5) LogInfo.begin_track("convert %s: head = %s, modifier = %s, env = %s", rawFormula, head, modifier, box.env); // Check binary/unary compatibility - boolean isNameFormula = (rawFormula instanceof ValueFormula) && (((ValueFormula)rawFormula).value instanceof NameValue); // Either binary or unary + boolean isNameFormula = (rawFormula instanceof ValueFormula) && (((ValueFormula) rawFormula).value instanceof NameValue); // Either binary or unary boolean needsBinary = (modifier != null); boolean providesBinary = rawFormula instanceof LambdaFormula || rawFormula instanceof ReverseFormula; if (!isNameFormula && needsBinary != providesBinary) { @@ -399,13 +525,13 @@ public class SparqlExecutor extends Executor { SparqlBlock block = new SparqlBlock(); if (rawFormula instanceof ValueFormula) { // e.g., fb:en.barack_obama or (number 3) - @SuppressWarnings({"unchecked"}) - ValueFormula formula = (ValueFormula)rawFormula; + @SuppressWarnings({ "unchecked" }) + ValueFormula formula = (ValueFormula) rawFormula; if (modifier != null) { // Binary predicate if (head.value == null) head.value = newVar(); if (modifier.value == null) modifier.value = newVar(); // Deal with primitive reverses (!fb:people.person.date_of_birth) - String property = ((NameValue)formula.value).id; + String property = ((NameValue) formula.value).id; PrimitiveFormula arg1, arg2; if (FreebaseInfo.isReverseProperty(property)) { arg1 = modifier.value; @@ -418,44 +544,45 @@ public class SparqlExecutor extends Executor { // Annoying logic to deal with dates. // If we have - // ?x fb:people.person.date_of_birth "2003"^^xsd:datetime, + // ?x fb:people.person.date_of_birth "2003"^xsd:datetime, // then create two statements: // ?x fb:people.person.date_of_birth ?v - // ?v = "2003"^^xsd:datetime [this needs to be transformed] + // ?v = "2003"^xsd:datetime [this needs to be transformed] if (!SparqlStatement.isOperator(property)) { if (arg2 instanceof ValueFormula) { - Value value = ((ValueFormula)arg2).value; + Value value = ((ValueFormula) arg2).value; if (value instanceof DateValue) { VariableFormula v = newVar(); - block.addStatement(v, "=", arg2); + addStatement(block, v, "=", arg2); arg2 = v; } } } - block.addStatement(arg1, property, arg2); + addStatement(block, arg1, property, arg2); } else { // Unary predicate unify(block, head, formula); } } else if (rawFormula instanceof VariableFormula) { - VariableFormula var = (VariableFormula)rawFormula; - PrimitiveFormula value = env.get(var); + VariableFormula var = (VariableFormula) rawFormula; + PrimitiveFormula value = box.env.get(var); if (value == null) - throw new RuntimeException("Unbound variable: " + var + ", env = " + env); + throw new RuntimeException("Unbound variable: " + var + ", env = " + box.env); unify(block, head, value); } else if (rawFormula instanceof NotFormula) { - NotFormula formula = (NotFormula)rawFormula; - block.add(new SparqlNot(convert(formula.child, head, null, env))); + NotFormula formula = (NotFormula) rawFormula; + block.add(new SparqlNot(convert(formula.child, head, null, box))); } else if (rawFormula instanceof MergeFormula) { - MergeFormula formula = (MergeFormula)rawFormula; + MergeFormula formula = (MergeFormula) rawFormula; switch (formula.mode) { case and: - block.add(convert(formula.child1, head, null, env)); - block.add(convert(formula.child2, head, null, env)); + block.add(convert(formula.child1, head, null, box)); + block.add(convert(formula.child2, head, null, box)); break; case or: SparqlUnion union = new SparqlUnion(); - union.add(convert(formula.child1, head, null, env)); - union.add(convert(formula.child2, head, null, env)); + ensureIsVar(box, block, head); + union.add(convert(formula.child1, head, null, box)); + union.add(convert(formula.child2, head, null, box)); block.add(union); break; default: @@ -463,109 +590,170 @@ public class SparqlExecutor extends Executor { } } else if (rawFormula instanceof JoinFormula) { // Join - JoinFormula formula = (JoinFormula)rawFormula; + JoinFormula formula = (JoinFormula) rawFormula; Ref intermediate = new Ref(); - block.add(convert(formula.child, intermediate, null, env)); - block.add(convert(formula.relation, head, intermediate, env)); + block.add(convert(formula.child, intermediate, null, box)); + block.add(convert(formula.relation, head, intermediate, box)); } else if (rawFormula instanceof ReverseFormula) { // Reverse - ReverseFormula formula = (ReverseFormula)rawFormula; - block.add(convert(formula.child, modifier, head, env)); // Switch modifier and head + ReverseFormula formula = (ReverseFormula) rawFormula; + block.add(convert(formula.child, modifier, head, box)); // Switch modifier and head } else if (rawFormula instanceof LambdaFormula) { - // Lambda (new environment) - LambdaFormula formula = (LambdaFormula)rawFormula; + // Lambda (new environment, same scope) + LambdaFormula formula = (LambdaFormula) rawFormula; if (modifier.value == null) modifier.value = newVar(); - Map newEnv = createNewEnv(formula.body, env); // Create new environment - newEnv.put(new VariableFormula(formula.var), modifier.value); // Map variable to modifier - block.add(convert(formula.body, head, null, newEnv)); + Box newBox = createNewBox(formula.body, box); // Create new environment + newBox.env.put(new VariableFormula(formula.var), modifier.value); // Map variable to modifier + block.add(convert(formula.body, head, null, newBox)); // Place pragmatic constraint that head != modifier (for symmetric relations like spouse) if (!opts.lambdaAllowDiagonals) - block.addStatement(head.value, "!=", modifier.value); + block.addStatement(head.value, "!=", modifier.value, false); + returnAsValuesMap(box, newBox); } else if (rawFormula instanceof MarkFormula) { - // Mark (new environment) - MarkFormula formula = (MarkFormula)rawFormula; + // Mark (new environment, same scope) + MarkFormula formula = (MarkFormula) rawFormula; if (head.value == null) head.value = newVar(); - Map newEnv = createNewEnv(formula.body, env); // Create new environment - newEnv.put(new VariableFormula(formula.var), head.value); // Map variable to head - block.add(convert(formula.body, head, null, newEnv)); + Box newBox = createNewBox(formula.body, box); // Create new environment + newBox.env.put(new VariableFormula(formula.var), head.value); // Map variable to head (ONLY difference with lambda) + block.add(convert(formula.body, head, null, newBox)); + returnAsValuesMap(box, newBox); } else if (rawFormula instanceof SuperlativeFormula) { // Superlative (new environment, close scope) - SuperlativeFormula formula = (SuperlativeFormula)rawFormula; + SuperlativeFormula formula = (SuperlativeFormula) rawFormula; - boolean useOrderBy = formula.rank != 1 || formula.count != 1; + int rank = Formulas.getInt(formula.rank); + int count = Formulas.getInt(formula.count); + + boolean useOrderBy = rank != 1 || count != 1; boolean isMax = formula.mode == SuperlativeFormula.Mode.argmax; if (useOrderBy) { - // Method 1: use ORDER BY (can deal with offset and limit, but can't be nested, and doesn't handle ties at the top) + // Method 1: use ORDER BY + // + can deal with offset and limit + // - but can't be nested + // - doesn't handle ties at the top // Recurse - Map newEnv = createNewEnv(formula.head, env); - SparqlBlock newBlock = convert(formula.head, head, null, newEnv); + Box newBox = createNewBox(formula.head, box); // Create new environment + SparqlBlock newBlock = convert(formula.head, head, null, newBox); Ref degree = new Ref(); - newBlock.add(convert(formula.relation, head, degree, newEnv)); + newBlock.add(convert(formula.relation, head, degree, newBox)); // Apply the aggregation operation - VariableFormula degreeVar = ensureIsVar(block, degree); + VariableFormula degreeVar = ensureIsVar(box, block, degree); // Force |degreeVar| to be selected as a variable. - env.put(new VariableFormula("degree"), degreeVar); - newEnv.put(new VariableFormula("degree"), degreeVar); + box.env.put(new VariableFormula("degree"), degreeVar); + newBox.env.put(new VariableFormula("degree"), degreeVar); - SparqlSelect select = closeExistentialScope(newBlock, head, newEnv, null, null); + SparqlSelect select = closeExistentialScope(newBlock, head, newBox); select.sortVars.add(isMax ? new VariableFormula(applyVar("DESC", degreeVar)) : degreeVar); - select.offset = formula.rank - 1; - select.limit = formula.count; + select.offset = rank - 1; + select.limit = count; block.add(select); } else { - // Method 2: use MAX (can be nested, handles ties at the top) + // Method 2: use MAX + // - can't deal with offset and limit + // + can be nested + // + handles ties at the top // (argmax 1 1 h r) ==> (h (r (mark degree (max ((reverse r) e))))) AggregateFormula.Mode mode = isMax ? AggregateFormula.Mode.max : AggregateFormula.Mode.min; Formula best = new MarkFormula("degree", new AggregateFormula(mode, new JoinFormula(new ReverseFormula(formula.relation), formula.head))); Formula transformed = new MergeFormula(MergeFormula.Mode.and, formula.head, new JoinFormula(formula.relation, best)); - //LogInfo.logs("TRANSFORMED: %s", transformed); - block.add(convert(transformed, head, null, env)); + if (opts.verbose >= 5) LogInfo.logs("TRANSFORMED: %s", transformed); + block.add(convert(transformed, head, null, box)); } } else if (rawFormula instanceof AggregateFormula) { // Aggregate (new environment, close scope) - AggregateFormula formula = (AggregateFormula)rawFormula; + AggregateFormula formula = (AggregateFormula) rawFormula; + ensureIsVar(box, block, head); // Recurse - Map newEnv = createNewEnv(formula.child, env); - Ref newHead = new Ref(newVar()); - SparqlBlock newBlock = convert(formula.child, newHead, null, newEnv); + Box newBox = createNewBox(formula.child, box); // Create new environment + Ref newHead = new Ref(newVar()); // Stores the aggregated value + SparqlBlock newBlock = convert(formula.child, newHead, null, newBox); - // Apply the aggregation operation - ensureIsVar(block, head); - String headAsValue = applyVar(formula.mode.toString(), (VariableFormula)newHead.value); - String headUnit = formula.mode == AggregateFormula.Mode.count ? NumberValue.unitless : getUnit(newBlock, (VariableFormula)newHead.value); - // Work around Virtuoso's inability to handle dates property: - // http://pablomendes.wordpress.com/2011/05/19/sparql-xsddate-weirdness/ - if ((formula.mode == AggregateFormula.Mode.min || formula.mode == AggregateFormula.Mode.max) && FreebaseInfo.DATE.equals(headUnit)) { - headAsValue = applyVar(formula.mode.toString(), applyVar("xsd:dateTime", (VariableFormula)newHead.value)); + VariableFormula var = (VariableFormula) head.value; // e.g., ?x - Ref intermediateHead = new Ref(); - block.add(closeExistentialScope(newBlock, intermediateHead, newEnv, headAsValue, headUnit)); - // Note: this statement introduces a filter, which only works if there are other constraints on |head| (e.g., this was created by doing an argmax, method 2) - block.addStatement(new VariableFormula(applyVar("xsd:dateTime", (VariableFormula)head.value)), "=", intermediateHead.value); + // Variable representing the aggregation + VariableFormula newVar = (VariableFormula) newHead.value; // e.g., ?y = COUNT(?x) + String headUnit = formula.mode == AggregateFormula.Mode.count ? NumberValue.unitless : unitsMap.get(newHead.value); + updateUnit(var, headUnit); + descriptionsMap.put(var, capitalize(formula.mode.toString())); + + // If do aggregation on dates, need to convert wrap with xsd:datetime + boolean specialDateHandling = + (formula.mode == AggregateFormula.Mode.min || formula.mode == AggregateFormula.Mode.max) && + FreebaseInfo.DATE.equals(headUnit); + + if (specialDateHandling) { + VariableFormula dateVar = new VariableFormula(var.name + "date"); + updateAsValues(newBox, dateVar, applyVar(formula.mode.toString(), SparqlUtils.dateTimeStr(newVar))); // ?dateVar AS max(xsd:datetime(?newVar)) + block.add(closeExistentialScope(newBlock, new Ref(dateVar), newBox)); + addStatement(block, new VariableFormula(SparqlUtils.dateTimeStr(var)), "=", dateVar); // Add xsd:datetime(?var) = ?dateVar on the outside. + // PROBLEM: (max ...) doesn't work by itself because the variable returned (var) is only involved in the previous construction, + // but this is fine in conjunction with other things like argmax. } else { - block.add(closeExistentialScope(newBlock, head, newEnv, headAsValue, headUnit)); + updateAsValues(newBox, var, applyVar(formula.mode.toString(), newVar)); // ?var AS COUNT(?newVar) + block.add(closeExistentialScope(newBlock, head, newBox)); } + } else if (rawFormula instanceof ArithmeticFormula) { // (+ (number 3) (number 5)) + ArithmeticFormula formula = (ArithmeticFormula) rawFormula; + Ref newHead1 = new Ref(); + Ref newHead2 = new Ref(); + block.add(convert(formula.child1, newHead1, null, box)); + block.add(convert(formula.child2, newHead2, null, box)); + if (head.value == null) head.value = newVar(); + VariableFormula var = (VariableFormula) head.value; + PrimitiveFormula var1 = newHead1.value; + PrimitiveFormula var2 = newHead2.value; + updateAsValues(box, var, applyOpVar(ArithmeticFormula.modeToString(formula.mode), var1, var2)); + String unit = unitsMap.get(var1); // Just take the unit from variable 1 + if (unit == null) unit = NumberValue.unitless; + String newUnit; + switch (formula.mode) { + case add: + newUnit = unit; + break; + case sub: + newUnit = FreebaseInfo.DATE.equals(unit) ? NumberValue.yearUnit : unit; + break; + default: + // Don't even try to get the unit right + newUnit = NumberValue.unitless; + break; + } + updateUnit(var, newUnit); + descriptionsMap.put(var, "Number"); // This is weak } else { throw new RuntimeException("Unhandled formula: " + rawFormula); } - if (opts.verbose >= 5) LogInfo.logs("return: head = %s, modifier = %s, env = %s", head, modifier, env); + if (opts.verbose >= 5) LogInfo.logs("return: head = %s, modifier = %s, env = %s", head, modifier, box.env); if (opts.verbose >= 5) LogInfo.end_track(); return block; } - // Copy |env|, but only keep the variables which are used in + private String capitalize(String s) { return Character.toUpperCase(s.charAt(0)) + s.substring(1); } + + // Copy |box|'s |env|, but only keep the variables which are used in |formula| (these are the free variables). // This is an important optimization for converting to SPARQL. - private Map createNewEnv(Formula formula, Map env) { - Map newEnv = new LinkedHashMap(); - for (VariableFormula key : env.keySet()) + private Box createNewBox(Formula formula, Box box) { + Box newBox = new Box(); + for (VariableFormula key : box.env.keySet()) if (Formulas.containsFreeVar(formula, key)) - newEnv.put(key, env.get(key)); - return newEnv; + newBox.env.put(key, box.env.get(key)); + return newBox; + } + + // Copy asValuesMap constraints from newBox to box. + // This is for when we create a new environment (newBox), but maintain the same scope, + // so we don't rely on closeExistentialScope to include the asValuesMap constraints. + private void returnAsValuesMap(Box box, Box newBox) { + for (Map.Entry e : newBox.asValuesMap.entrySet()) { + if (box.asValuesMap.containsKey(e.getKey())) + throw new RuntimeException("Copying asValuesMap involves overwriting: " + box + " <- " + newBox); + box.asValuesMap.put(e.getKey(), e.getValue()); + } } private void unify(SparqlBlock block, Ref head, PrimitiveFormula value) { @@ -574,13 +762,30 @@ public class SparqlExecutor extends Executor { head.value = value; } else { // |head| is already set, so add a constraint that it equals |value|. - block.addStatement(head.value, "=", value); + // This happens when the logical form is just a single entity (e.g., fb:en.barack_obama). + addStatement(block, head.value, "=", value); + if (head.value instanceof VariableFormula && value instanceof ValueFormula && ((ValueFormula) value).value instanceof NameValue) + addEntityStatement(block, (VariableFormula) head.value); } } // Helper functions private String applyVar(String func, VariableFormula var) { return applyVar(func, var.name); } - private String applyVar(String func, String var) { return func + "(" + var + ")"; } + private String applyVar(String func, String var) { + if (func.equals("count")) var = "DISTINCT " + var; + return func + "(" + var + ")"; + } + + private String applyOpVar(String func, PrimitiveFormula var1, PrimitiveFormula var2) { + // Special function for taking the difference between dates. + LogInfo.logs("%s %s", var1, var2); + if (func.equals("-") && (var1 instanceof VariableFormula) && FreebaseInfo.DATE.equals(unitsMap.get(var1))) + return "bif:datediff(\"year\"," + SparqlUtils.dateTimeStr(var2) + "," + SparqlUtils.dateTimeStr(var1) + ")"; + else if (func.equals("+") && (var1 instanceof VariableFormula) && FreebaseInfo.DATE.equals(unitsMap.get(var1))) + return "bif:dateadd(\"year\"," + Formulas.getString(var2) + "," + SparqlUtils.dateTimeStr(var1) + ")"; // date + number + else + return '(' + Formulas.getString(var1) + ' ' + func + ' ' + Formulas.getString(var2) + ')'; + } private VariableFormula newVar() { numVars++; @@ -595,6 +800,7 @@ public class SparqlExecutor extends Executor { final Formula formula; final List selectVars; final List units; + final List header; public ValuesExtractor(boolean beginTrack, Formula formula, Converter converter) { this.beginTrack = beginTrack; @@ -602,30 +808,43 @@ public class SparqlExecutor extends Executor { this.selectVars = Lists.newArrayList(); this.units = Lists.newArrayList(); + this.header = Lists.newArrayList(); for (SparqlSelect.Var var : converter.query.selectVars) { if (var.isAuxiliary) continue; this.selectVars.add(var.var.name); this.units.add(var.unit); + this.header.add(var.description); } } // |results| is (result (binding (uri ...)) ...) or (result (binding (literal ...)) ...) - List extract(NodeList results) { + Value extract(NodeList results) { // For each result (row in a table)... if (beginTrack && opts.verbose >= 2) { LogInfo.begin_track("%d results", results.getLength()); + if (opts.returnTable) LogInfo.logs("Header: %s", header); } - List values = new ArrayList(); + + List firstValues = new ArrayList(); // If not returning a table + List> rows = new ArrayList>(); // If returning table + for (int i = 0; i < results.getLength(); i++) { - Value value = nodeToValue(results.item(i)); - values.add(value); - if (beginTrack && opts.verbose >= 2) LogInfo.logs("%s", value); + List row = nodeToValue(results.item(i)); + if (opts.returnTable) + rows.add(row); + else + firstValues.add(row.get(0)); + if (beginTrack && opts.verbose >= 2) LogInfo.logs("Row %d: %s", i, row); } if (beginTrack && opts.verbose >= 2) LogInfo.end_track(); - return values; + + if (opts.returnTable) + return new TableValue(header, rows); + else + return new ListValue(firstValues); } - private Value nodeToValue(Node result) { + private List nodeToValue(Node result) { NodeList bindings = ((Element) result).getElementsByTagName("binding"); // For each variable in selectVars, we're going to keep track of an |id| @@ -639,7 +858,7 @@ public class SparqlExecutor extends Executor { // For each binding j (contributes some information to one column)... for (int j = 0; j < bindings.getLength(); j++) { - Element binding = (Element)bindings.item(j); + Element binding = (Element) bindings.item(j); String var = "?" + binding.getAttribute("name"); int col; @@ -667,7 +886,7 @@ public class SparqlExecutor extends Executor { if (unit == null) { value = new NameValue(id, description); } else if (unit.equals(FreebaseInfo.DATE)) { - value = DateValue.parseDateValue(description); + value = description == null ? null : DateValue.parseDateValue(description); } else if (unit.equals(FreebaseInfo.TEXT)) { value = new StringValue(description); } else if (unit.equals(FreebaseInfo.BOOLEAN)) { @@ -682,12 +901,7 @@ public class SparqlExecutor extends Executor { } row.add(value); } - - // Either keep just the head of the row or the entire row. - if (opts.returnTable) - return new ListValue(row); - else - return row.get(0); + return row; } } @@ -713,7 +927,7 @@ public class SparqlExecutor extends Executor { public String formulasPath; } - public static void main(String[] args) throws NumberFormatException, IOException { + public static void main(String[] args) throws IOException { OptionsParser parser = new OptionsParser(); MainOptions mainOpts = new MainOptions(); parser.registerAll(new Object[]{"SparqlExecutor", SparqlExecutor.opts, "FreebaseInfo", FreebaseInfo.opts, "main", mainOpts}); @@ -723,13 +937,13 @@ public class SparqlExecutor extends Executor { SparqlExecutor executor = new SparqlExecutor(); if (mainOpts.formula != null) { - LogInfo.logs("%s", executor.execute(Formulas.fromLispTree(LispTree.proto.parseFromString(mainOpts.formula))).value); + LogInfo.logs("%s", executor.execute(Formulas.fromLispTree(LispTree.proto.parseFromString(mainOpts.formula)), null).value); } if (mainOpts.formulasPath != null) { Iterator trees = LispTree.proto.parseFromFile(mainOpts.formulasPath); while (trees.hasNext()) { - LogInfo.logs("%s", executor.execute(Formulas.fromLispTree(trees.next())).value); + LogInfo.logs("%s", executor.execute(Formulas.fromLispTree(trees.next()), null).value); } } @@ -739,11 +953,3 @@ public class SparqlExecutor extends Executor { LogInfo.end_track(); } } - -class BadFormulaException extends Exception { - public static final long serialVersionUID = 86586128316354597L; - String message; - public BadFormulaException(String message) { this.message = message; } - @Override - public String toString() { return message; } -} diff --git a/src/edu/stanford/nlp/sempre/SparqlExpr.java b/src/edu/stanford/nlp/sempre/freebase/SparqlExpr.java similarity index 77% rename from src/edu/stanford/nlp/sempre/SparqlExpr.java rename to src/edu/stanford/nlp/sempre/freebase/SparqlExpr.java index c4a2521..0aa99c4 100644 --- a/src/edu/stanford/nlp/sempre/SparqlExpr.java +++ b/src/edu/stanford/nlp/sempre/freebase/SparqlExpr.java @@ -1,8 +1,9 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.freebase; import java.util.*; import com.google.common.collect.Lists; import fig.basic.*; +import edu.stanford.nlp.sempre.*; /** * Utilities for constructing SPARQL expressions. @@ -18,18 +19,20 @@ class SparqlBlock implements SparqlExpr { public SparqlBlock add(SparqlExpr expr) { if (expr instanceof SparqlBlock) - this.children.addAll(((SparqlBlock)expr).children); + this.children.addAll(((SparqlBlock) expr).children); else this.children.add(expr); return this; } - boolean isPrimitiveType(PrimitiveFormula formula) { - if (!(formula instanceof ValueFormula)) return false; - Value value = ((ValueFormula)formula).value; - if (!(value instanceof NameValue)) return false; - String id = ((NameValue)value).id; + private String getId(PrimitiveFormula formula) { + if (!(formula instanceof ValueFormula)) return null; + Value value = ((ValueFormula) formula).value; + if (!(value instanceof NameValue)) return null; + return ((NameValue) value).id; + } + private boolean isPrimitiveType(String id) { if (FreebaseInfo.BOOLEAN.equals(id)) return true; if (FreebaseInfo.INT.equals(id)) return true; if (FreebaseInfo.FLOAT.equals(id)) return true; @@ -38,28 +41,21 @@ class SparqlBlock implements SparqlExpr { return false; } - void addOptionalStatement(PrimitiveFormula arg1, String property, PrimitiveFormula arg2) { addStatement(arg1, property, arg2, true); } - void addStatement(PrimitiveFormula arg1, String property, PrimitiveFormula arg2) { addStatement(arg1, property, arg2, false); } void addStatement(PrimitiveFormula arg1, String property, PrimitiveFormula arg2, boolean optional) { - // Ignore statements like + if (!property.startsWith("fb:") && !SparqlStatement.isOperator(property) && !SparqlStatement.isSpecialFunction(property) && !SparqlStatement.isIndependent(property)) + throw new RuntimeException("Invalid SPARQL property: " + property); + // Ignore statements like: // ?x fb:type.object.type fb:type.datetime - // because ?x is not actually an entity. - if (property.equals(FreebaseInfo.TYPE) && isPrimitiveType(arg2)) return; - //if (property.equals(FreebaseInfo.CONTAINED_BY)) { - // TODO This doesn't work - if (false) { - int TRANSITIVE_MAX = 3; // FIXME PARAM - String name1 = ((VariableFormula) arg1).name; - String name2 = ((VariableFormula) arg2).name; - String option_string = String.format("OPTION (TRANSITIVE, t_max (%d), t_in (%s), t_out (%s), t_step('step_no') as %s_%s_dist )", TRANSITIVE_MAX, name1, name2, name1, name2.substring(1)); - add(new SparqlStatement(arg1, property, arg2, optional, option_string)); + // because we should have already captured the semantics using other formulas that involve ?x. + if (property.equals(FreebaseInfo.TYPE)) { + String id = getId(arg2); + if (isPrimitiveType(id) || FreebaseInfo.ANY.equals(id)) return; } - else - add(new SparqlStatement(arg1, property, arg2, optional)); + if (SparqlStatement.isIndependent(property)) return; // Nothing connecting arg1 and arg2 + add(new SparqlStatement(arg1, property, arg2, optional)); } @Override public String toString() { - //return "{ " + StrUtils.join(children, " . ") + " }"; List strings = Lists.newArrayList(); for (SparqlExpr expr : children) { if (expr instanceof SparqlSelect) @@ -78,17 +74,19 @@ class SparqlSelect implements SparqlExpr { public final String asValue; // for COUNT(?x3) as ?x2 public final String unit; // Specifies the types of the variable (used to parse back the results) public final boolean isAuxiliary; // Whether this is supporting information (e.g., names) + public final String description; // Human-friendly for display - public Var(VariableFormula var, String asValue, String unit, boolean isAuxiliary) { + public Var(VariableFormula var, String asValue, String unit, boolean isAuxiliary, String description) { this.var = var; this.asValue = asValue; this.unit = unit; this.isAuxiliary = isAuxiliary; + this.description = description; } @Override public String toString() { if (asValue == null) return var.name; - return asValue + " AS " + var.name; + return "(" + asValue + " AS " + var.name + ")"; } } @@ -154,33 +152,39 @@ class SparqlStatement implements SparqlExpr { this.options = options; } + public static boolean isIndependent(String relation) { return relation.equals(":"); } + public static boolean isOperator(String relation) { return relation.equals("=") || relation.equals("!=") || relation.equals("<") || relation.equals(">") || relation.equals("<=") || relation.equals(">="); } + public static boolean isSpecialFunction(String relation) { + return relation.equals("STRSTARTS") || relation.equals("STRENDS"); + } + public String simpleString() { // Workaround for annoying dates: // http://answers.semanticweb.com/questions/947/dbpedia-sparql-endpoint-xsddate-comparison-weirdness - if (arg2 instanceof ValueFormula && ((ValueFormula)arg2).value instanceof DateValue) { + if (arg2 instanceof ValueFormula && ((ValueFormula) arg2).value instanceof DateValue) { if (isOperator(relation)) { if (relation.equals("=")) { // (= (date 2000 -1 -1)) really means (>= (2000 -1 -1)) and (< (2001 -1 -1)) - DateValue startDate = (DateValue)((ValueFormula)arg2).value; + DateValue startDate = (DateValue) ((ValueFormula) arg2).value; DateValue endDate = advance(startDate); return SparqlUtils.dateTimeStr(arg1) + " >= " + SparqlUtils.dateTimeStr(new ValueFormula(startDate)) + ") . FILTER (" + SparqlUtils.dateTimeStr(arg1) + " < " + SparqlUtils.dateTimeStr(new ValueFormula(endDate)); } if (relation.equals("<=")) { // (<= (date 2000 -1 -1)) really means (< (date 2001 -1 -1)) - DateValue startDate = (DateValue)((ValueFormula)arg2).value; + DateValue startDate = (DateValue) ((ValueFormula) arg2).value; DateValue endDate = advance(startDate); return SparqlUtils.dateTimeStr(arg1) + " < " + SparqlUtils.dateTimeStr(new ValueFormula(endDate)); } if (relation.equals(">")) { // (> (date 2000 -1 -1)) really means >= (date 2001 -1 -1) - DateValue startDate = (DateValue)((ValueFormula)arg2).value; + DateValue startDate = (DateValue) ((ValueFormula) arg2).value; DateValue endDate = advance(startDate); return SparqlUtils.dateTimeStr(arg1) + " >= " + SparqlUtils.dateTimeStr(new ValueFormula(endDate)); } @@ -194,7 +198,7 @@ class SparqlStatement implements SparqlExpr { } private DateValue advance(DateValue date) { - // TODO: deal with carrying over + // TODO(pliang): deal with carrying over if (date.day != -1) return new DateValue(date.year, date.month, date.day + 1); if (date.month != -1) return new DateValue(date.year, date.month + 1, -1); return new DateValue(date.year + 1, -1, -1); @@ -202,7 +206,7 @@ class SparqlStatement implements SparqlExpr { public String toString() { String result; - if (relation.equals("STRSTARTS")) { // Special functions + if (isSpecialFunction(relation)) { // Special functions result = "FILTER (" + relation + "(" + SparqlUtils.plainStr(arg1) + "," + SparqlUtils.plainStr(arg2) + "))"; } else if (isOperator(relation)) { result = "FILTER (" + simpleString() + ")"; @@ -219,24 +223,26 @@ class SparqlStatement implements SparqlExpr { } } -class SparqlUtils { +final class SparqlUtils { + private SparqlUtils() { } + public static String dateTimeStr(PrimitiveFormula formula) { - return "xsd:dateTime(" + plainStr(formula) + ")"; + return "xsd:datetime(" + plainStr(formula) + ")"; } public static String plainStr(PrimitiveFormula formula) { if (formula instanceof VariableFormula) return ((VariableFormula) formula).name; - Value value = ((ValueFormula)formula).value; + Value value = ((ValueFormula) formula).value; if (value instanceof StringValue) { - String s = ((StringValue)value).value; + String s = ((StringValue) value).value; return "\"" + s.replaceAll("\"", "\\\\\"") + "\"" + (s.equals("en") ? "" : "@en"); } if (value instanceof NameValue) return ((NameValue) value).id; if (value instanceof NumberValue) return ((NumberValue) value).value + ""; if (value instanceof DateValue) { - DateValue date = (DateValue)value; + DateValue date = (DateValue) value; if (date.month == -1) return "\"" + date.year + "\"" + "^^xsd:datetime"; if (date.day == -1) return "\"" + date.year + "-" + date.month + "\"" + "^^xsd:datetime"; return "\"" + date.year + "-" + date.month + "-" + date.day + "\"" + "^^xsd:datetime"; diff --git a/src/edu/stanford/nlp/sempre/Stemmer.java b/src/edu/stanford/nlp/sempre/freebase/Stemmer.java similarity index 87% rename from src/edu/stanford/nlp/sempre/Stemmer.java rename to src/edu/stanford/nlp/sempre/freebase/Stemmer.java index 45a34b8..aad6ea2 100644 --- a/src/edu/stanford/nlp/sempre/Stemmer.java +++ b/src/edu/stanford/nlp/sempre/freebase/Stemmer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.freebase; import org.tartarus.snowball.ext.PorterStemmer; diff --git a/src/edu/stanford/nlp/sempre/TextToTextMatcher.java b/src/edu/stanford/nlp/sempre/freebase/TextToTextMatcher.java similarity index 67% rename from src/edu/stanford/nlp/sempre/TextToTextMatcher.java rename to src/edu/stanford/nlp/sempre/freebase/TextToTextMatcher.java index 3bb9cf9..1cad53c 100644 --- a/src/edu/stanford/nlp/sempre/TextToTextMatcher.java +++ b/src/edu/stanford/nlp/sempre/freebase/TextToTextMatcher.java @@ -1,10 +1,10 @@ -package edu.stanford.nlp.sempre; +package edu.stanford.nlp.sempre.freebase; -import edu.stanford.nlp.sempre.fbalignment.lexicons.TokenLevelMatchFeatures; +import edu.stanford.nlp.sempre.freebase.lexicons.TokenLevelMatchFeatures; +import edu.stanford.nlp.sempre.*; import edu.stanford.nlp.stats.ClassicCounter; import edu.stanford.nlp.stats.Counter; import edu.stanford.nlp.stats.Counters; -import edu.stanford.nlp.util.StringUtils; import fig.basic.LogInfo; import fig.basic.Option; @@ -30,11 +30,9 @@ public class TextToTextMatcher { if (textToTextMatcher == null) textToTextMatcher = new TextToTextMatcher(); return textToTextMatcher; } - //private WordDistance wordDistance; private Stemmer stemmer; public TextToTextMatcher() { - //wordDistance = WordDistance.getSingleton(); stemmer = new Stemmer(); } @@ -45,58 +43,38 @@ public class TextToTextMatcher { return res; } + // JONATHAN - changing to not have edit distance since it is inefficient private void extractWordSimilarityFeatures(List exampleTokens, List examplePosTags, Set fbDescs, FeatureVector vector) { if (!FeatureExtractor.containsDomain("wordSim")) return; - int minEditDistance = Integer.MAX_VALUE; - //ExtremeValueWrapper simWrapper = WordDistance.buildDistanceWrapper(); - - //String minWordDistFbToken = "", minWordDistTextToken = ""; - String minEditDistFbToken = "", minEditDistTextToken = ""; - //String similarityPos = ""; - + boolean match = false; + // collect fb tokens + List candidateFbTokens = new ArrayList<>(); for (String fbDesc : fbDescs) { List fbDescTokens = FbFormulasInfo.BinaryFormulaInfo.tokenizeFbDescription(fbDesc); for (String fbDescToken : fbDescTokens) { if (fbDescToken.length() <= 2) continue; - - for (int i = 0; i < exampleTokens.size(); ++i) { - - String pos = examplePosTags.get(i); - String textToken = exampleTokens.get(i); - if (pos.startsWith("NN") || (pos.startsWith("VB") && !pos.equals("VBD-AUX")) || pos.equals("JJ")) { - - int currEditDist = StringUtils.editDistance(textToken, fbDescToken); - //double currWordDist = wordDistance.score(textToken, fbDescToken); - - if (currEditDist < minEditDistance) { - minEditDistance = currEditDist; - minEditDistFbToken = fbDescToken; - minEditDistTextToken = textToken; - } -// if (simWrapper.add(currWordDist)) { -// minWordDistFbToken = fbDescToken; -// minWordDistTextToken = textToken; -// similarityPos = examplePosTags.get(i).substring(0, 1); -// } - } - + candidateFbTokens.add(fbDesc); + } + } + // check if there is a token that is an fb token candidate + for (int i = 0; i < exampleTokens.size(); ++i) { + String pos = examplePosTags.get(i); + String textToken = exampleTokens.get(i); + if (pos.startsWith("NN") || (pos.startsWith("VB") && !pos.equals("VBD-AUX")) || pos.equals("JJ")) { + if (candidateFbTokens.contains(textToken)) { + match = true; + break; } } } - minEditDistance = Math.min(minEditDistance, 15); - if (opts.verbose >= 3) { - LogInfo.logs("Min edit distance [%s,%s]: %s", minEditDistFbToken, minEditDistTextToken, minEditDistance); - //LogInfo.logs("Extreme similarity [%s,%s]: %s", minWordDistFbToken, minWordDistTextToken, simWrapper.distance); - } - vector.addWithBias("wordSim", "binary_edit_distance", minEditDistance); -// if (!similarityPos.equals("")) -// vector.addWithBias("wordSim", "binary_word_sim_" + similarityPos, simWrapper.distance); + if (match) + vector.add("wordSim", "FbTokenExTokenMatch"); } public boolean existsTokenMatch(List exampleTokens, List exampleLemmas, Set fbDescs) { - //generate stems + // generate stems List exampleStems = new ArrayList(); for (String token : exampleTokens) exampleStems.add(stemmer.stem(token)); @@ -105,7 +83,7 @@ public class TextToTextMatcher { Counter stemFeatures = new ClassicCounter(); for (String fbDescription : fbDescs) { List fbDescTokens = FbFormulasInfo.BinaryFormulaInfo.tokenizeFbDescription(fbDescription); - List fbDescStems = new ArrayList(); + List fbDescStems = new ArrayList<>(); for (String fbDescToken : fbDescTokens) fbDescStems.add(stemmer.stem(fbDescToken)); @@ -121,16 +99,16 @@ public class TextToTextMatcher { private void extractTokenMatchFeatures(List exampleTokens, List exampleLemmas, Set fbDescs, FeatureVector vector) { if (!FeatureExtractor.containsDomain("tokenMatch")) return; - //generate stems - List exampleStems = new ArrayList(); + // generate stems + List exampleStems = new ArrayList<>(); for (String token : exampleTokens) exampleStems.add(stemmer.stem(token)); - Counter tokenFeatures = new ClassicCounter(); - Counter stemFeatures = new ClassicCounter(); + Counter tokenFeatures = new ClassicCounter<>(); + Counter stemFeatures = new ClassicCounter<>(); for (String fbDescription : fbDescs) { List fbDescTokens = FbFormulasInfo.BinaryFormulaInfo.tokenizeFbDescription(fbDescription); - List fbDescStems = new ArrayList(); + List fbDescStems = new ArrayList<>(); for (String fbDescToken : fbDescTokens) fbDescStems.add(stemmer.stem(fbDescToken)); diff --git a/src/edu/stanford/nlp/sempre/freebase/UnaryLexicon.java b/src/edu/stanford/nlp/sempre/freebase/UnaryLexicon.java new file mode 100644 index 0000000..3738164 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/UnaryLexicon.java @@ -0,0 +1,193 @@ +package edu.stanford.nlp.sempre.freebase; + +import com.google.common.base.Strings; +import edu.stanford.nlp.io.IOUtils; +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.UnaryFormulaInfo; +import edu.stanford.nlp.sempre.freebase.lexicons.EntrySource; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.LexiconValue; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.UnaryLexicalEntry; +import fig.basic.LogInfo; +import fig.basic.MapUtils; +import fig.basic.Option; +import fig.basic.StopWatchSet; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +/** + * lexicon for unaries: "city"-->fb:location.citytown + * Loads the lexicon from a file and has some features + */ +public final class UnaryLexicon { + + public static class Options { + @Option(gloss = "Number of results return by the lexicon") + public int maxEntries = 1000; + @Option(gloss = "Path to unary lexicon file") + public String unaryLexiconFilePath = "lib/fb_data/7/unaryInfoStringAndAlignment.txt"; + @Option(gloss = "Threshold for filtering unaries") + public int unaryFilterThreshold = 5; + @Option(gloss = "Verbosity") public int verbose = 0; + } + + public static Options opts = new Options(); + + private static UnaryLexicon unaryLexicon; + public static UnaryLexicon getInstance() { + if (unaryLexicon == null) unaryLexicon = new UnaryLexicon(); + return unaryLexicon; + } + + private Map> lexemeToEntryList = new HashMap<>(); + + public static final String INTERSECTION = "intersection"; + public static final String NL_SIZE = "nl_size"; + public static final String FB_SIZE = "fb_size"; + + private UnaryLexicon() { + if (Strings.isNullOrEmpty(opts.unaryLexiconFilePath)) + throw new RuntimeException("Missing unary lexicon file"); + read(); + sortLexiconEntries(); + } + + private void sortLexiconEntries() { + Comparator comparator = new UnaryLexicalEntryComparator(); + for (List uEntries: lexemeToEntryList.values()) { + Collections.sort(uEntries, comparator); + } + } + + private void read() { + LogInfo.begin_track("Loading unary lexicon file " + opts.unaryLexiconFilePath); + for (String line : IOUtils.readLines(opts.unaryLexiconFilePath)) { + LexiconValue lv = Json.readValueHard(line, LexiconValue.class); + addEntry(lv.lexeme, lv.source, lv.formula, lv.features); + } + LogInfo.log("Number of lexemes: " + lexemeToEntryList.size()); + LogInfo.end_track(); + } + + private void addEntry(String nl, String source, Formula formula, Map featureMap) { + + FbFormulasInfo ffi = FbFormulasInfo.getSingleton(); + if (ffi.getUnaryInfo(formula) != null) { + UnaryFormulaInfo uInfo = ffi.getUnaryInfo(formula); + UnaryLexicalEntry uEntry = new UnaryLexicalEntry(nl, nl, new TreeSet<>(uInfo.descriptions), formula, EntrySource.parseSourceDesc(source), + uInfo.popularity, new TreeMap<>(featureMap), uInfo.types); + MapUtils.addToList(lexemeToEntryList, nl, uEntry); + } else { + if (opts.verbose >= 3) LogInfo.warnings("Missing info for unary: %s ", formula); + } + } + + public void save(String outFile) throws IOException { + + PrintWriter writer = IOUtils.getPrintWriter(outFile); + for (String nl : lexemeToEntryList.keySet()) { + for (UnaryLexicalEntry uEntry : lexemeToEntryList.get(nl)) { + LexiconValue lv = new LexiconValue(nl, uEntry.formula, uEntry.source.toString(), uEntry.alignmentScores); + writer.println(Json.writeValueAsStringHard(lv)); + } + } + writer.close(); + } + + public List lookupEntries(String textDesc) throws IOException { + + List entries = lexemeToEntryList.get(textDesc.toLowerCase()); + if (entries != null) { + List res = new ArrayList<>(); + for (int i = 0; i < Math.min(entries.size(), opts.maxEntries); ++i) { + if (valid(entries.get(i))) + res.add(entries.get(i)); + } + return res; + } + return Collections.emptyList(); + } + + /** Checks if an entry is valid (e.g. we filter if intersection is too small) */ + private boolean valid(UnaryLexicalEntry lexicalEntry) { + return (lexicalEntry.source != EntrySource.ALIGNMENT || + MapUtils.getDouble(lexicalEntry.alignmentScores, INTERSECTION, 0.0) >= opts.unaryFilterThreshold); + } + + public void sortLexiconByFeedback(Params params) { + StopWatchSet.begin("UnaryLexicon.sortLexiconByFeedback"); + LogInfo.log("Number of entries: " + lexemeToEntryList.size()); + UnaryLexEntrybyFeaturesComparator comparator = new UnaryLexEntrybyFeaturesComparator(params); + for (String lexeme : lexemeToEntryList.keySet()) { + Collections.sort(lexemeToEntryList.get(lexeme), comparator); + if (LexiconFn.opts.verbose > 0) { + LogInfo.logs("Sorted list for lexeme=%s", lexeme); + for (UnaryLexicalEntry uEntry : lexemeToEntryList.get(lexeme)) { + FeatureVector fv = new FeatureVector(); + LexiconFn.getUnaryEntryFeatures(uEntry, fv); + LogInfo.logs("Entry=%s, dotprod=%s", uEntry, fv.dotProduct(comparator.params)); + } + } + } + StopWatchSet.end(); + } + + public class UnaryLexEntrybyFeaturesComparator implements Comparator { + public final Params params; + public UnaryLexEntrybyFeaturesComparator(Params params) { + this.params = params; + } + @Override + public int compare(UnaryLexicalEntry entry1, UnaryLexicalEntry entry2) { + + FeatureVector features1 = new FeatureVector(); + FeatureVector features2 = new FeatureVector(); + LexiconFn.getUnaryEntryFeatures(entry1, features1); + LexiconFn.getUnaryEntryFeatures(entry2, features2); + double score1 = features1.dotProduct(params); + double score2 = features2.dotProduct(params); + if (score1 > score2) return -1; + if (score1 < score2) return +1; + // back off to usual thing + double entry1Intersection = MapUtils.getDouble(entry1.alignmentScores, INTERSECTION, 0.0); + double entry2Intersection = MapUtils.getDouble(entry2.alignmentScores, INTERSECTION, 0.0); + if (entry1Intersection > entry2Intersection) + return -1; + if (entry1Intersection < entry2Intersection) + return 1; + if (entry1.popularity > entry2.popularity) + return -1; + if (entry1.popularity < entry2.popularity) + return 1; + return 0; + } + } + + public static class UnaryLexicalEntryComparator implements Comparator { + + public static final String INTERSECTION = "intersection"; + + @Override + public int compare(UnaryLexicalEntry entry1, UnaryLexicalEntry entry2) { + + double entry1Intersection = MapUtils.getDouble(entry1.alignmentScores, INTERSECTION, 0.0); + double entry2Intersection = MapUtils.getDouble(entry2.alignmentScores, INTERSECTION, 0.0); + if (entry1Intersection > entry2Intersection) + return -1; + if (entry1Intersection < entry2Intersection) + return 1; + if (entry1.popularity > entry2.popularity) + return -1; + if (entry1.popularity < entry2.popularity) + return 1; + // todo - this is to break ties - make more efficient + int stringComparison = entry1.formula.toString().compareTo(entry2.formula.toString()); + if (stringComparison < 0) + return -1; + if (stringComparison > 0) + return +1; + return 0; + } + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/Utils.java b/src/edu/stanford/nlp/sempre/freebase/Utils.java index 1f0b0b9..6434918 100644 --- a/src/edu/stanford/nlp/sempre/freebase/Utils.java +++ b/src/edu/stanford/nlp/sempre/freebase/Utils.java @@ -9,7 +9,9 @@ import java.io.PrintWriter; import java.util.HashMap; import java.util.Map; -public class Utils { +public final class Utils { + private Utils() { } + public static final String ttlPrefix = "@prefix fb: ."; // Somewhat of a crude approximation. @@ -22,13 +24,13 @@ public class Utils { // Convert a string from the ns: namespace to the fb: namespace. // "ns:en.barack_obama" => "fb:en.barack_obama" - public static String ns_to_fb(String s) { + public static String nsToFb(String s) { if (s.startsWith("ns:")) return "fb:" + s.substring(3); return s; } // "\"/en/distributive_writing\"" => "fb:en.distributive_writing" - public static String string_to_rdf(String arg2) { + public static String stringToRdf(String arg2) { if (!arg2.startsWith("\"/") || !arg2.endsWith("\"")) throw new RuntimeException("Bad: " + arg2); return "fb:" + arg2.substring(2, arg2.length() - 1).replaceAll("/", "."); @@ -38,21 +40,21 @@ public class Utils { if (!line.endsWith(".")) return null; String[] tokens = line.substring(0, line.length() - 1).split("\t"); if (tokens.length != 3) return null; - tokens[0] = Utils.ns_to_fb(tokens[0]); - tokens[1] = Utils.ns_to_fb(tokens[1]); - tokens[2] = Utils.ns_to_fb(tokens[2]); + tokens[0] = Utils.nsToFb(tokens[0]); + tokens[1] = Utils.nsToFb(tokens[1]); + tokens[2] = Utils.nsToFb(tokens[2]); return tokens; } - + public static int parseInt(String arg2) { - if(!arg2.endsWith("^^xsd:int")) + if (!arg2.endsWith("^^xsd:int")) throw new RuntimeException("Arg2 is not a valid integer: " + arg2); int closingQuoteIndex = arg2.lastIndexOf('"'); return Integer.parseInt(arg2.substring(1, closingQuoteIndex)); } - + public static String parseStr(String arg2) { - if(!arg2.endsWith("@en")) + if (!arg2.endsWith("@en")) throw new RuntimeException("Arg2 is not a valid String: " + arg2); int closingQuoteIndex = arg2.lastIndexOf('"'); return arg2.substring(1, closingQuoteIndex); diff --git a/src/edu/stanford/nlp/sempre/freebase/build.xml b/src/edu/stanford/nlp/sempre/freebase/build.xml new file mode 100644 index 0000000..0112ffe --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/build.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/edu/stanford/nlp/sempre/fbalignment/index/FbEntityIndexer.java b/src/edu/stanford/nlp/sempre/freebase/index/FbEntityIndexer.java similarity index 96% rename from src/edu/stanford/nlp/sempre/fbalignment/index/FbEntityIndexer.java rename to src/edu/stanford/nlp/sempre/freebase/index/FbEntityIndexer.java index 74cf47f..b32ff9e 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/index/FbEntityIndexer.java +++ b/src/edu/stanford/nlp/sempre/freebase/index/FbEntityIndexer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.index; +package edu.stanford.nlp.sempre.freebase.index; import edu.stanford.nlp.io.IOUtils; import fig.basic.LogInfo; @@ -16,7 +16,7 @@ import java.io.File; import java.io.IOException; public class FbEntityIndexer { - + private final IndexWriter indexer; private String nameFile; @@ -28,8 +28,8 @@ public class FbEntityIndexer { IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44 , indexingStrategy.equals("exact") ? new KeywordAnalyzer() : new StandardAnalyzer(Version.LUCENE_44)); config.setOpenMode(OpenMode.CREATE); config.setRAMBufferSizeMB(256.0); - indexer = new IndexWriter(new SimpleFSDirectory(new File(outputDir)),config); - + indexer = new IndexWriter(new SimpleFSDirectory(new File(outputDir)), config); + this.nameFile = namefile; } @@ -37,7 +37,6 @@ public class FbEntityIndexer { * Index the datadump file * * @throws IOException - * @throws FreebaseDataDumpException */ public void index() throws IOException { diff --git a/src/edu/stanford/nlp/sempre/fbalignment/index/FbEntitySearcher.java b/src/edu/stanford/nlp/sempre/freebase/index/FbEntitySearcher.java similarity index 93% rename from src/edu/stanford/nlp/sempre/fbalignment/index/FbEntitySearcher.java rename to src/edu/stanford/nlp/sempre/freebase/index/FbEntitySearcher.java index 1a3f7ca..2d2591a 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/index/FbEntitySearcher.java +++ b/src/edu/stanford/nlp/sempre/freebase/index/FbEntitySearcher.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.index; +package edu.stanford.nlp.sempre.freebase.index; import fig.basic.LogInfo; import fig.basic.StopWatch; @@ -24,7 +24,7 @@ import java.util.List; import java.util.regex.Pattern; public class FbEntitySearcher { - + private final QueryParser queryParser; private final IndexSearcher indexSearcher; private int numOfDocs = 50; @@ -47,7 +47,7 @@ public class FbEntitySearcher { LogInfo.log("Opened index with " + indexReader.numDocs() + " documents."); this.numOfDocs = numOfDocs; - LogInfo.end_track("Constructing Searcher"); + LogInfo.end_track(); } public synchronized List searchDocs(String question) throws IOException, ParseException { @@ -74,21 +74,21 @@ public class FbEntitySearcher { public static void main(String[] args) throws IOException, ParseException { - Pattern QUIT = + Pattern quit = Pattern.compile("quit|exit|q|bye", Pattern.CASE_INSENSITIVE); FbEntitySearcher searcher = new FbEntitySearcher(args[0], 10000, args[1]); BufferedReader is = new BufferedReader(new InputStreamReader(System.in)); StopWatch watch = new StopWatch(); - for (; ; ) { + while (true) { System.out.print("Search> "); String question = is.readLine().trim(); - if (QUIT.matcher(question).matches()) { + if (quit.matcher(question).matches()) { System.out.println("Quitting."); break; } if (question.equals("")) continue; - + watch.reset(); watch.start(); List docs = searcher.searchDocs(question); @@ -101,7 +101,7 @@ public class FbEntitySearcher { "Name: " + doc.get(FbIndexField.TEXT.fieldName()) + "\t" + "Popularity: " + doc.get(FbIndexField.POPULARITY.fieldName())); } - LogInfo.logs("Number of docs: %s, Time: %s",docs.size(), watch); + LogInfo.logs("Number of docs: %s, Time: %s", docs.size(), watch); } } } diff --git a/src/edu/stanford/nlp/sempre/fbalignment/index/FbIndexField.java b/src/edu/stanford/nlp/sempre/freebase/index/FbIndexField.java similarity index 83% rename from src/edu/stanford/nlp/sempre/fbalignment/index/FbIndexField.java rename to src/edu/stanford/nlp/sempre/freebase/index/FbIndexField.java index 7d8d773..31d96a9 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/index/FbIndexField.java +++ b/src/edu/stanford/nlp/sempre/freebase/index/FbIndexField.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.index; +package edu.stanford.nlp.sempre.freebase.index; public enum FbIndexField { @@ -7,7 +7,7 @@ public enum FbIndexField { ID("id"), TYPES("types"), POPULARITY("popularity"); - + private final String fieldName; FbIndexField(String fieldName) { diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntrySource.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/EntrySource.java similarity index 85% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntrySource.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/EntrySource.java index 392a12d..ff2ea9c 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntrySource.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/EntrySource.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; +package edu.stanford.nlp.sempre.freebase.lexicons; public enum EntrySource { @@ -8,7 +8,8 @@ public enum EntrySource { HARD_CODED("HARD"), LUCENE("LUCENE"), GRAPHPROP("GRAPHPROP"), - FBSEARCH("FBSEARCH"); + FBSEARCH("FBSEARCH"), + FEEDBACK("FEEDBACK"); EntrySource(String source) { this.source = source; @@ -37,6 +38,8 @@ public enum EntrySource { return GRAPHPROP; if (desc.equals("FBSEARCH")) return FBSEARCH; + if (desc.equals("FEEDBACK")) + return FEEDBACK; throw new RuntimeException("Description is not legal: " + desc); } diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/ExtremeValueWrapper.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/ExtremeValueWrapper.java similarity index 92% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/ExtremeValueWrapper.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/ExtremeValueWrapper.java index 6f58b1a..6c76853 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/ExtremeValueWrapper.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/ExtremeValueWrapper.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; +package edu.stanford.nlp.sempre.freebase.lexicons; public abstract class ExtremeValueWrapper { public double distance; diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/LexicalEntry.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/LexicalEntry.java similarity index 81% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/LexicalEntry.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/LexicalEntry.java index 0f7698a..fc4dcb8 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/LexicalEntry.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/LexicalEntry.java @@ -1,13 +1,15 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; +package edu.stanford.nlp.sempre.freebase.lexicons; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; + import edu.stanford.nlp.sempre.Formula; +import edu.stanford.nlp.sempre.SemType; +import edu.stanford.nlp.sempre.SemTypeHierarchy; import edu.stanford.nlp.stats.ClassicCounter; import edu.stanford.nlp.stats.Counter; import edu.stanford.nlp.util.StringUtils; import fig.basic.LispTree; -import fig.basic.MapUtils; import java.util.HashSet; import java.util.Map; @@ -16,13 +18,14 @@ import java.util.TreeMap; public class LexicalEntry { - public String textDescription; //the query as submitted to the lexicon - public String normalizedTextDesc; //the query after normalization - public Set fbDescriptions; //descriptions matching the formula - public Formula formula; + public final String textDescription; // the query as submitted to the lexicon + public final String normalizedTextDesc; // the query after normalization + public final Set fbDescriptions; // descriptions matching the formula + public final Formula formula; public EntrySource source; - public double popularity; - public double distance; + public final double popularity; + public final double distance; + public LexicalEntry(String textDescription, String normalizedTextDesc, Set fbDescriptions, Formula formula, EntrySource source, double popularity, double distance) { this.textDescription = textDescription; @@ -71,26 +74,31 @@ public class LexicalEntry { return Math.min(15, distance); } + // Input: set of types coming from the lexicon {fb:common.topic, fb:people.person, ...} + // Output: remove any element which is in the transitive closure. {fb:people.person, ...} + // TODO(pliang): replace HashSet with something lighter weight. + public static SemType setToType(Set types) { + Set resultTypes = new HashSet<>(types); + for (String entityType : types) { + for (String supertype : SemTypeHierarchy.singleton.getSupertypes(entityType)) { + if (!supertype.equals(entityType)) + resultTypes.remove(supertype); + } + } + return SemType.newUnionSemType(resultTypes); + } + public static class BinaryLexicalEntry extends LexicalEntry { public String expectedType1; public String expectedType2; public String unitId = ""; public String unitDescription = ""; - public Map alignmentScores; - public String fullLexeme; //the lexeme as it is in the alignment without some normalization applied before uploading the lexicon - - public BinaryLexicalEntry(String textDescription, String normalizedTextDesc, Set fbDescriptions, Formula formula, - EntrySource source, double popularity, String expectedType1, String expectedType2, Map alignmentScores, String fullLexeme) { - super(textDescription, normalizedTextDesc, fbDescriptions, formula, source, popularity, computeEditDistance(textDescription, fbDescriptions)); - this.expectedType1 = expectedType1; - this.expectedType2 = expectedType2; - this.alignmentScores = alignmentScores; - this.fullLexeme = fullLexeme; - } + public Map alignmentScores; + public String fullLexeme; // the lexeme as it is in the alignment without some normalization applied before uploading the lexicon public BinaryLexicalEntry(String textDescription, String normalizedTextDesc, Set fbDescriptions, Formula formula, EntrySource source, double popularity, - String expectedType1, String expectedType2, String unitId, String unitDesc, Map alignmentScores, String fullLexeme) { + String expectedType1, String expectedType2, String unitId, String unitDesc, Map alignmentScores, String fullLexeme) { super(textDescription, normalizedTextDesc, fbDescriptions, formula, source, popularity, computeEditDistance(textDescription, fbDescriptions)); this.expectedType1 = expectedType1; this.expectedType2 = expectedType2; @@ -170,37 +178,21 @@ public class LexicalEntry { String rightContext = fullLexeme.substring(fullLexeme.indexOf(normalizedTextDesc) + normalizedTextDesc.length()).trim(); return rightContext.split("\\s+"); } - - public double jaccard() { - double intersection = MapUtils.getDouble(alignmentScores, BinaryLexicon.INTERSECTION, 0.0); - if (intersection < 2.01) - intersection = 0.0; - double nlSize = MapUtils.getDouble(alignmentScores, BinaryLexicon.NL_TYPED, 0.0); - double fbSize = MapUtils.getDouble(alignmentScores, BinaryLexicon.FB_TYPED, 0.0); - return intersection / (nlSize + fbSize - intersection + 5); - } - - /** rids all counts and keeps jaccard only */ - public void retainJaccardOnly() { - double jaccard = jaccard(); - alignmentScores.clear(); - alignmentScores.put("jaccard", jaccard); - } } public static class EntityLexicalEntry extends LexicalEntry { - public Set types = new HashSet(); - public Counter tokenEditDistanceFeatures; + public Set types = new HashSet<>(); + public SemType type; + public Counter entityFeatures; - public EntityLexicalEntry(String textDescription, String normalizedTextDesc, Set fbDescriptions, Formula formula, EntrySource source, double popularity, double distance, Set types, Counter tokenEditDistanceFeatures) { + public EntityLexicalEntry(String textDescription, String normalizedTextDesc, Set fbDescriptions, + Formula formula, EntrySource source, double popularity, double distance, Set types, + Counter entityFeatures) { super(textDescription, normalizedTextDesc, fbDescriptions, formula, source, popularity, distance); this.types = types; - this.tokenEditDistanceFeatures = tokenEditDistanceFeatures; - } - - public Set getTypes() { - return types; + this.type = setToType(types); + this.entityFeatures = entityFeatures; } public String toString() { @@ -211,19 +203,17 @@ public class LexicalEntry { public static class UnaryLexicalEntry extends LexicalEntry { public Set types = new HashSet(); - public Map alignmentScores; + public SemType type; + public Map alignmentScores; public UnaryLexicalEntry(String textDescription, String normalizedTextDesc, Set fbDescriptions, Formula formula, EntrySource source, double popularity, - Map alignmentScores, Set types) { + Map alignmentScores, Set types) { super(textDescription, normalizedTextDesc, fbDescriptions, formula, source, popularity, computeEditDistance(textDescription, fbDescriptions)); this.types = types; + this.type = setToType(types); this.alignmentScores = alignmentScores; } - public Set getTypes() { - return types; - } - String stringRepn; public String toString() { if (stringRepn == null) @@ -242,13 +232,13 @@ public class LexicalEntry { @JsonProperty public String lexeme; @JsonProperty public Formula formula; @JsonProperty public String source; - @JsonProperty public Map features; + @JsonProperty public Map features; @JsonCreator public LexiconValue(@JsonProperty("normLexeme") String lexeme, @JsonProperty("formula") Formula formula, @JsonProperty("source") String source, - @JsonProperty("features") Map features) { + @JsonProperty("features") Map features) { this.lexeme = lexeme; this.formula = formula; this.source = source; @@ -271,14 +261,14 @@ public class LexicalEntry { return tree; } - static Map featureMapFromLispTree(LispTree tree) { - Map featureMap = new TreeMap(); + static Map featureMapFromLispTree(LispTree tree) { + Map featureMap = new TreeMap(); for (int i = 0; i < tree.children.size(); i++) featureMap.put(tree.child(i).child(0).value, Double.parseDouble(tree.child(i).child(1).value)); return featureMap; } - static LispTree featureMapToLispTree(Map featureMap) { + static LispTree featureMapToLispTree(Map featureMap) { LispTree tree = LispTree.proto.newList(); for (String feature : featureMap.keySet()) tree.addChild(LispTree.proto.newList(feature, "" + featureMap.get(feature))); @@ -337,7 +327,7 @@ public class LexicalEntry { EntrySource source = EntrySource.parseSourceDesc(tree.child(i++).value); double popularity = Double.parseDouble(tree.child(i++).value); Double.parseDouble(tree.child(i++).value); - Map alignmentScores = featureMapFromLispTree(tree.child(i++)); + Map alignmentScores = featureMapFromLispTree(tree.child(i++)); Set types = setFromLispTree(tree.child(i++)); return new LexicalEntry.UnaryLexicalEntry( textDescription, normalizedTextDesc, fbDescriptions, formula, source, @@ -349,12 +339,12 @@ public class LexicalEntry { Formula formula = Formula.fromString(tree.child(i++).value); EntrySource source = EntrySource.parseSourceDesc(tree.child(i++).value); double popularity = Double.parseDouble(tree.child(i++).value); - Double.parseDouble(tree.child(i++).value); //this is computed in the constructor so need not save it + Double.parseDouble(tree.child(i++).value); // this is computed in the constructor so need not save it String expectedType1 = tree.child(i++).value; String expectedType2 = tree.child(i++).value; String unitId = tree.child(i++).value; String unitDescription = tree.child(i++).value; - Map alignmentScores = featureMapFromLispTree(tree.child(i++)); + Map alignmentScores = featureMapFromLispTree(tree.child(i++)); String fullLexeme = tree.child(i++).value; return new LexicalEntry.BinaryLexicalEntry( textDescription, normalizedTextDesc, fbDescriptions, formula, source, @@ -380,8 +370,7 @@ public class LexicalEntry { result.addChild("" + entry.popularity); result.addChild("" + entry.distance); result.addChild(setToLispTree(entry.types)); - result.addChild(counterToLispTree(entry.tokenEditDistanceFeatures)); - + result.addChild(counterToLispTree(entry.entityFeatures)); } else if (rawEntry instanceof LexicalEntry.UnaryLexicalEntry) { LexicalEntry.UnaryLexicalEntry entry = (LexicalEntry.UnaryLexicalEntry) rawEntry; result.addChild("unary"); diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/TokenLevelMatchFeatures.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/TokenLevelMatchFeatures.java similarity index 96% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/TokenLevelMatchFeatures.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/TokenLevelMatchFeatures.java index 4c3ec35..b40b735 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/TokenLevelMatchFeatures.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/TokenLevelMatchFeatures.java @@ -1,6 +1,6 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons; +package edu.stanford.nlp.sempre.freebase.lexicons; -import edu.stanford.nlp.sempre.fbalignment.utils.FileUtils; +import edu.stanford.nlp.sempre.freebase.utils.FileUtils; import edu.stanford.nlp.stats.ClassicCounter; import edu.stanford.nlp.stats.Counter; import fig.basic.LogInfo; @@ -9,7 +9,8 @@ import fig.basic.Option; import java.util.Arrays; import java.util.List; -public class TokenLevelMatchFeatures { +public final class TokenLevelMatchFeatures { + private TokenLevelMatchFeatures() { } public static class Options { @Option(gloss = "Verbose") public int verbose = 0; @@ -188,7 +189,7 @@ public class TokenLevelMatchFeatures { for (int i = 0; i < source.size(); ++i) { for (int j = 0; j < target.size(); ++j) { - if (target.get(j).length() <= 2) //do not match very short words + if (target.get(j).length() <= 2) // do not match very short words continue; int matchLength = findLongestMatch(source, target, i, j, strict); diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/BinaryNormalizer.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/BinaryNormalizer.java similarity index 97% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/BinaryNormalizer.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/BinaryNormalizer.java index 94061e8..ccea095 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/BinaryNormalizer.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/BinaryNormalizer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers; +package edu.stanford.nlp.sempre.freebase.lexicons.normalizers; import edu.stanford.nlp.util.ArrayUtils; diff --git a/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/EntryNormalizer.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/EntryNormalizer.java new file mode 100644 index 0000000..dbaf090 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/EntryNormalizer.java @@ -0,0 +1,5 @@ +package edu.stanford.nlp.sempre.freebase.lexicons.normalizers; + +public interface EntryNormalizer { + String normalize(String str); +} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/IdentityNormalizer.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/IdentityNormalizer.java similarity index 67% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/IdentityNormalizer.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/IdentityNormalizer.java index 16f29d8..a7181e2 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/IdentityNormalizer.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/IdentityNormalizer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers; +package edu.stanford.nlp.sempre.freebase.lexicons.normalizers; public class IdentityNormalizer implements EntryNormalizer { diff --git a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/PrepDropNormalizer.java b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/PrepDropNormalizer.java similarity index 92% rename from src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/PrepDropNormalizer.java rename to src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/PrepDropNormalizer.java index 2a51e6e..4a59a54 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/PrepDropNormalizer.java +++ b/src/edu/stanford/nlp/sempre/freebase/lexicons/normalizers/PrepDropNormalizer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers; +package edu.stanford.nlp.sempre.freebase.lexicons.normalizers; import edu.stanford.nlp.util.ArrayUtils; diff --git a/src/edu/stanford/nlp/sempre/test/FbFormulasTest.java b/src/edu/stanford/nlp/sempre/freebase/test/FbFormulasTest.java similarity index 66% rename from src/edu/stanford/nlp/sempre/test/FbFormulasTest.java rename to src/edu/stanford/nlp/sempre/freebase/test/FbFormulasTest.java index ca21ae9..58c10b4 100644 --- a/src/edu/stanford/nlp/sempre/test/FbFormulasTest.java +++ b/src/edu/stanford/nlp/sempre/freebase/test/FbFormulasTest.java @@ -1,8 +1,8 @@ -package edu.stanford.nlp.sempre.test; +package edu.stanford.nlp.sempre.freebase.test; -import edu.stanford.nlp.sempre.FbFormulasInfo; -import edu.stanford.nlp.sempre.FbFormulasInfo.BinaryFormulaInfo; -import edu.stanford.nlp.sempre.FbFormulasInfo.UnaryFormulaInfo; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.BinaryFormulaInfo; +import edu.stanford.nlp.sempre.freebase.FbFormulasInfo.UnaryFormulaInfo; import edu.stanford.nlp.sempre.Formula; import org.testng.annotations.Test; @@ -14,33 +14,33 @@ public class FbFormulasTest { public void formulaInfo() { FbFormulasInfo infoRepos = FbFormulasInfo.getSingleton(); - //1 + // 1 BinaryFormulaInfo bInfo = infoRepos.getBinaryInfo(Formula.fromString("(lambda x (!fb:education.education.specialization (!fb:education.field_of_study.students_majoring (var x))))")); assertEquals(521.0, bInfo.popularity, 0.0001); assertEquals("fb:education.field_of_study", bInfo.expectedType1); assertEquals("fb:education.field_of_study", bInfo.expectedType2); boolean contains = bInfo.descriptions.contains("specialization") && bInfo.descriptions.contains("students majoring in this field"); assertEquals(true, contains); - //2 - bInfo=infoRepos.getBinaryInfo(Formula.fromString("!fb:broadcast.content.broadcast")); - assertEquals(4838.0, bInfo.popularity,0.0001); + // 2 + bInfo = infoRepos.getBinaryInfo(Formula.fromString("!fb:broadcast.content.broadcast")); + assertEquals(4838.0, bInfo.popularity, 0.0001); assertEquals("fb:broadcast.broadcast", bInfo.expectedType1); assertEquals("fb:broadcast.content", bInfo.expectedType2); contains = bInfo.descriptions.contains("broadcasts"); - assertEquals(true, contains); - //3 + assertEquals(true, contains); + // 3 UnaryFormulaInfo uInfo = infoRepos.getUnaryInfo(Formula.fromString("(fb:type.object.type fb:location.country)")); - assertEquals(574.0, uInfo.popularity,0.0001); + assertEquals(574.0, uInfo.popularity, 0.0001); contains = uInfo.descriptions.contains("country") && uInfo.descriptions.contains("empire"); assertEquals(true, contains); - assertEquals("fb:location.country",uInfo.types.iterator().next()); - //4 + assertEquals("fb:location.country", uInfo.types.iterator().next()); + // 4 uInfo = infoRepos.getUnaryInfo(Formula.fromString("(fb:people.person.profession fb:en.wrestler)")); - assertEquals(1449.0, uInfo.popularity,0.0001); - contains = uInfo.descriptions.contains("wrestler") + assertEquals(1449.0, uInfo.popularity, 0.0001); + contains = uInfo.descriptions.contains("wrestler") && uInfo.descriptions.contains("professional wrestler") && uInfo.descriptions.contains("pro wrestler"); assertEquals(true, contains); - assertEquals("fb:people.person",uInfo.types.iterator().next()); + assertEquals("fb:people.person", uInfo.types.iterator().next()); } } diff --git a/src/edu/stanford/nlp/sempre/freebase/test/FreebaseInfoTest.java b/src/edu/stanford/nlp/sempre/freebase/test/FreebaseInfoTest.java new file mode 100644 index 0000000..3351fb8 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/test/FreebaseInfoTest.java @@ -0,0 +1,24 @@ +package edu.stanford.nlp.sempre.freebase.test; + +import edu.stanford.nlp.sempre.freebase.FreebaseInfo; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertEquals; + +/** + * Test FreebaseInfo. + * @author Percy Liang + */ +public class FreebaseInfoTest { + @Test public void units() { + FreebaseInfo info = FreebaseInfo.getSingleton(); + assertEquals(FreebaseInfo.ENTITY, info.getUnit1("fb:people.person.place_of_birth")); + assertEquals(FreebaseInfo.ENTITY, info.getUnit2("fb:people.person.place_of_birth")); + + assertEquals(FreebaseInfo.ENTITY, info.getUnit1("fb:people.person.date_of_birth")); + assertEquals(FreebaseInfo.DATE, info.getUnit2("fb:people.person.date_of_birth")); + + assertEquals(FreebaseInfo.ENTITY, info.getUnit1("fb:people.person.height_meters")); + assertEquals("fb:en.meter", info.getUnit2("fb:people.person.height_meters")); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/test/FreebaseSemTypeTest.java b/src/edu/stanford/nlp/sempre/freebase/test/FreebaseSemTypeTest.java new file mode 100644 index 0000000..8f53b8f --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/test/FreebaseSemTypeTest.java @@ -0,0 +1,67 @@ +package edu.stanford.nlp.sempre.freebase.test; + +import org.testng.annotations.Test; + +import edu.stanford.nlp.sempre.SemType; +import edu.stanford.nlp.sempre.SemTypeHierarchy; +import edu.stanford.nlp.sempre.freebase.*; +import fig.basic.LispTree; + +/** + * Test type system on Freebase schema. + * @author Percy Liang + */ +public class FreebaseSemTypeTest { + // For testing + private static SemType T(String str) { + return SemType.fromLispTree(LispTree.proto.parseFromString(str)); + } + + private static void verifyEquals(SemType predType, SemType wantedType) { + if (!predType.toString().equals(wantedType.toString())) + throw new RuntimeException(String.format("Wanted %s, but got %s", wantedType, predType)); + } + + private static void verifyMeet(String t1, String t2) { verifyMeet(t1, t2, t2); } + private static void verifyMeet(String t1, String t2, String t) { + verifyEquals(T(t1).meet(T(t2)), T(t)); + verifyEquals(T(t2).meet(T(t1)), T(t)); + } + + @Test public void simpleSemType() { + FreebaseInfo.getSingleton(); // Load Freebase type hierarchy + SemTypeHierarchy.opts.failOnUnknownTypes = false; + verifyMeet("city", "city"); + verifyMeet("city", "country", "(union)"); + verifyMeet("city", "(union city country)", "city"); + verifyMeet("(union city country river)", "(union city country)"); + + verifyEquals(T("(-> city fb:type.int)").apply(T("(union city country)")), T("fb:type.int")); + verifyEquals(T("(-> city fb:type.int fb:type.float)").apply(T("(union city country)")).apply(T("fb:type.int")), T("fb:type.float")); + verifyEquals(T("fb:type.datetime").apply(T("fb:common.topic")), T("(union)")); + verifyEquals(T("(-> fb:location.citytown fb:type.datetime)").apply(T("fb:location.location")), T("fb:type.datetime")); + verifyEquals(T("(-> fb:location.location fb:type.datetime)").apply(T("fb:location.citytown")), T("fb:type.datetime")); + + verifyMeet("(-> fb:location.location fb:type.number)", "(-> fb:location.location fb:type.float)"); + verifyMeet("fb:common.topic", "fb:location.location"); + verifyMeet("fb:type.any", "fb:type.boolean"); + verifyMeet("fb:type.any", "fb:type.number"); + verifyMeet("fb:type.any", "fb:type.datetime"); + verifyMeet("fb:type.any", "fb:type.cvt"); + verifyMeet("fb:type.any", "fb:type.text"); + verifyMeet("fb:type.any", "fb:location.location"); + verifyMeet("fb:type.any", "fb:common.topic"); + + verifyMeet("fb:common.topic", "fb:common.topic"); + verifyMeet("top", "(-> t t)"); + verifyMeet("top", "fb:type.datetime"); + verifyMeet("top", "(union a b)"); + + verifyMeet("(-> (-> a b) top)", "(-> top (-> a b))", "(-> (-> a b) (-> a b))"); + verifyMeet("(-> (union city country) person)", "(-> city (union person dog))", "(-> city person)"); + } + + public static void main(String[] args) { + new FreebaseSemTypeTest().simpleSemType(); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/test/FreebaseTypeInferenceTest.java b/src/edu/stanford/nlp/sempre/freebase/test/FreebaseTypeInferenceTest.java new file mode 100644 index 0000000..b2dc1b4 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/test/FreebaseTypeInferenceTest.java @@ -0,0 +1,92 @@ +package edu.stanford.nlp.sempre.freebase.test; + +import static org.testng.AssertJUnit.assertEquals; + +import org.testng.annotations.Test; + +import edu.stanford.nlp.sempre.*; +import edu.stanford.nlp.sempre.freebase.*; + +/** + * Test type inference on Freebase schema. + * @author Percy Liang + */ +public class FreebaseTypeInferenceTest { + // For testing + private static Formula F(String str) { return Formula.fromString(str); } + private static SemType T(String str) { return SemType.fromString(str); } + private static SemType FT(String str) { return TypeInference.inferType(F(str)); } + + void check(String fstr, String tstr) { + System.out.println("check " + fstr + " " + tstr); + assertEquals(T(tstr).toString(), FT(fstr).toString()); + } + + @Test public void simpleSemType() { + // TODO(pliang): replace with NullTypeLookup()? + FreebaseTypeLookup.opts.entityTypesPath = null; // Disable entity lookup + TypeInference.setTypeLookup(new FreebaseTypeLookup()); + + check("(fb:location.location.area (>= (number 200)))", "fb:location.location"); + + check("(number 3)", "fb:type.number"); + check("(string foo)", "fb:type.text"); + check("(date 1981 1 1)", "fb:type.datetime"); + check("fb:en.barack_obama", "fb:common.topic"); // Don't have getEntityTypes + check("fb:people.person.place_of_birth", "(-> fb:location.location fb:people.person)"); + + // Join + check("(fb:type.object.type fb:location.location)", "fb:location.location"); + check("(fb:people.person.place_of_birth (fb:type.object.type fb:location.location))", "fb:people.person"); + check("(!fb:people.person.place_of_birth (fb:type.object.type fb:location.location))", "(union)"); + + // Merge + check("(and (fb:type.object.type fb:common.topic) (fb:people.person.place_of_birth fb:en.seattle))", "fb:people.person"); + check("(and (fb:type.object.type fb:location.location) (fb:people.person.place_of_birth fb:en.seattle))", "(union)"); + + // Mark + check("(mark x (fb:people.person.parents (var x)))", "fb:people.person"); + check("(mark x (fb:people.person.place_of_birth (var x)))", "(union)"); + + // Lambda + check("(lambda x (fb:people.person.place_of_birth (var x)))", "(-> fb:location.location fb:people.person)"); + check("(lambda x (!fb:people.person.place_of_birth (var x)))", "(-> fb:people.person fb:location.location)"); + check("(lambda x (fb:people.person.place_of_birth (var x)))", "(-> fb:location.location fb:people.person)"); + check("(lambda x (!fb:people.person.place_of_birth (var x)))", "(-> fb:people.person fb:location.location)"); + check("(lambda x (!fb:people.person.profession (fb:people.person.place_of_birth (var x))))", "(-> fb:location.location fb:people.profession)"); + check("(lambda b ((var b) (fb:type.object.type fb:people.person)))", "(-> (-> fb:people.person top) top)"); + // Note: and the other way doesn't work, since we don't propagate everything. + check("(lambda b (and (fb:type.object.type fb:location.location) ((var b) (fb:type.object.type fb:people.person))))", "(-> (-> fb:people.person fb:location.location) fb:location.location)"); + check("(lambda x (lambda y ((var x) (var y))))", "(-> (-> top top) (-> top top))"); + check("(lambda x (lambda x (fb:people.person.place_of_birth (var x))))", "(-> top (-> fb:location.location fb:people.person))"); // No variable capture + + // Aggregation + check("(lambda x (not (var x)))", "(-> fb:type.any fb:type.any)"); + check("(lambda x (count (var x)))", "(-> fb:type.any fb:type.number)"); + check("(lambda x (count (fb:people.person.place_of_birth (var x))))", "(-> fb:location.location fb:type.number)"); + + // Arithmetic + check("(+ (number 3) (number 4))", "fb:type.number"); + check("(+ (date 1981 1 1) (string 4))", "(union)"); + check("(- (date 1982 1 1) (date 1981 1 1))", "fb:type.datetime"); // Future: should be a different duration type + + // Reverse + check("(reverse fb:people.person.place_of_birth)", "(-> fb:people.person fb:location.location)"); + + // Superlative + check("(argmax 1 1 (fb:type.object.type fb:people.person) fb:people.person.date_of_birth)", "fb:people.person"); + check("(argmax 1 1 (fb:type.object.type fb:common.topic) fb:people.person.date_of_birth)", "fb:people.person"); + check("(argmax 1 1 (fb:type.object.type fb:common.topic) (reverse (lambda x (number 3))))", "fb:common.topic"); + check("(lambda x (lambda y (argmax 1 1 (var x) (var y))))", "(-> fb:type.any (-> (-> (union fb:type.number fb:type.datetime) fb:type.any) fb:type.any))"); + + // Call + check("(call Math.cos (number 0))", "fb:type.float"); + check("(call Math.cos (string abc))", "(union)"); + check("(lambda x (lambda y (call .concat (var x) (var y))))", "(-> fb:type.text (-> fb:type.text fb:type.text))"); + check("(lambda x (call .length (var x)))", "(-> fb:type.text fb:type.int)"); + } + + public static void main(String[] args) { + new FreebaseTypeInferenceTest().simpleSemType(); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/test/LexiconTest.java b/src/edu/stanford/nlp/sempre/freebase/test/LexiconTest.java new file mode 100644 index 0000000..6177946 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/test/LexiconTest.java @@ -0,0 +1,85 @@ +package edu.stanford.nlp.sempre.freebase.test; + +import edu.stanford.nlp.sempre.freebase.BinaryLexicon; +import edu.stanford.nlp.sempre.freebase.lexicons.EntrySource; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.BinaryLexicalEntry; +import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.UnaryLexicalEntry; +import edu.stanford.nlp.sempre.freebase.UnaryLexicon; +import fig.basic.LogInfo; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.List; + +import static org.testng.AssertJUnit.assertEquals; + +public class LexiconTest { + @Test + public void unary() throws IOException { + UnaryLexicon.opts.unaryLexiconFilePath = "unittest-files/unaryInfoStringAndAlignment.txt"; + UnaryLexicon unary = UnaryLexicon.getInstance(); + boolean existsAlignment = false, existsStringMatch = false; + double popularity = 0.0; + double intersection = 0.0; + + List entries = unary.lookupEntries("continent"); + LogInfo.logs("Num of unary entries for 'continent': %s", entries.size()); + for (UnaryLexicalEntry entry : entries) { + if (entry.formula.toString().equals("(fb:type.object.type fb:location.continent)")) { + if (entry.source == EntrySource.ALIGNMENT) { + existsAlignment = true; + intersection = entry.alignmentScores.get(UnaryLexicon.INTERSECTION); + } else if (entry.source == EntrySource.STRING_MATCH) { + existsStringMatch = true; + popularity = entry.popularity; + } + } + } + assertEquals(true, existsAlignment); + assertEquals(true, existsStringMatch); + assertEquals(7.0, popularity, 0.0001); + assertEquals(5.0, intersection, 0.0001); + + existsAlignment = false; + existsStringMatch = false; + popularity = 0.0; + intersection = 0.0; + entries = unary.lookupEntries("lawyer"); + LogInfo.logs("Num of unary entries for 'lawyer': %s", entries.size()); + for (UnaryLexicalEntry entry : entries) { + if (entry.formula.toString().equals("(fb:people.person.profession fb:en.attorney)")) { + if (entry.source == EntrySource.ALIGNMENT) { + existsAlignment = true; + intersection = entry.alignmentScores.get(UnaryLexicon.INTERSECTION); + } else if (entry.source == EntrySource.STRING_MATCH) { + existsStringMatch = true; + popularity = entry.popularity; + } + } + } + assertEquals(true, existsAlignment); + assertEquals(true, existsStringMatch); + assertEquals(12282.0, popularity, 0.0001); // Based on 93.exec (full Freebase) + assertEquals(26.0, intersection, 0.0001); + } + + @Test + public void binary() throws IOException { + BinaryLexicon.opts.binaryLexiconFilesPath = "unittest-files/binaryInfoStringAndAlignment.txt"; + BinaryLexicon.opts.keyToSortBy = BinaryLexicon.INTERSECTION; + + BinaryLexicon lexicon = BinaryLexicon.getInstance(); + List entries = lexicon.lookupEntries("bear in"); + LogInfo.logs("Num of binary entries for 'bear in': %s", entries.size()); + BinaryLexicalEntry top = entries.get(0); + assertEquals("people born here", top.fbDescriptions.iterator().next()); + assertEquals("!fb:location.location.people_born_here", top.formula.toString()); + assertEquals("ALIGNMENT", top.source.toString()); + assertEquals(759773.0, top.popularity, 0.00001); // Based on 93.exec (full Freebase) + assertEquals("fb:people.person", top.expectedType1); + assertEquals("fb:location.location", top.expectedType2); + assertEquals(16184.0, top.alignmentScores.get("FB_typed_size"), 0.0001); + assertEquals(13856.0, top.alignmentScores.get("Intersection_size_typed"), 0.0001); + assertEquals(15765.0, top.alignmentScores.get("NL_typed_size"), 0.0001); + } +} diff --git a/src/edu/stanford/nlp/sempre/test/PrepDropNormalizerTest.java b/src/edu/stanford/nlp/sempre/freebase/test/PrepDropNormalizerTest.java similarity index 52% rename from src/edu/stanford/nlp/sempre/test/PrepDropNormalizerTest.java rename to src/edu/stanford/nlp/sempre/freebase/test/PrepDropNormalizerTest.java index 0df6ee2..56585a0 100644 --- a/src/edu/stanford/nlp/sempre/test/PrepDropNormalizerTest.java +++ b/src/edu/stanford/nlp/sempre/freebase/test/PrepDropNormalizerTest.java @@ -1,6 +1,6 @@ -package edu.stanford.nlp.sempre.test; +package edu.stanford.nlp.sempre.freebase.test; -import edu.stanford.nlp.sempre.fbalignment.lexicons.normalizers.PrepDropNormalizer; +import edu.stanford.nlp.sempre.freebase.lexicons.normalizers.PrepDropNormalizer; import org.testng.annotations.Test; import static org.testng.AssertJUnit.assertEquals; @@ -11,13 +11,13 @@ import static org.testng.AssertJUnit.assertEquals; */ public class PrepDropNormalizerTest { - + @Test public void normalization() { PrepDropNormalizer normalizer = new PrepDropNormalizer(); - assertEquals("interested",normalizer.normalize("interested in")); + assertEquals("interested", normalizer.normalize("interested in")); assertEquals("interested", normalizer.normalize("interested at")); - assertEquals("blow up",normalizer.normalize("blow up in")); - assertEquals("blow up the",normalizer.normalize("blow up the to")); + assertEquals("blow up", normalizer.normalize("blow up in")); + assertEquals("blow up the", normalizer.normalize("blow up the to")); } } diff --git a/src/edu/stanford/nlp/sempre/test/SparqlExecutorTest.java b/src/edu/stanford/nlp/sempre/freebase/test/SparqlExecutorTest.java similarity index 78% rename from src/edu/stanford/nlp/sempre/test/SparqlExecutorTest.java rename to src/edu/stanford/nlp/sempre/freebase/test/SparqlExecutorTest.java index fbee369..2d5eb68 100644 --- a/src/edu/stanford/nlp/sempre/test/SparqlExecutorTest.java +++ b/src/edu/stanford/nlp/sempre/freebase/test/SparqlExecutorTest.java @@ -1,11 +1,11 @@ -package edu.stanford.nlp.sempre.test; +package edu.stanford.nlp.sempre.freebase.test; import java.util.List; import edu.stanford.nlp.sempre.Executor; import edu.stanford.nlp.sempre.Formulas; import edu.stanford.nlp.sempre.Value; import edu.stanford.nlp.sempre.ListValue; -import edu.stanford.nlp.sempre.SparqlExecutor; +import edu.stanford.nlp.sempre.freebase.SparqlExecutor; import fig.basic.LispTree; import fig.basic.LogInfo; import org.testng.annotations.Test; @@ -61,9 +61,9 @@ public class SparqlExecutorTest { } protected static void runFormula(SparqlExecutor executor, String formula, ValuesChecker checker) { - Executor.Response response = executor.execute(Formulas.fromLispTree(LispTree.proto.parseFromString(formula))); + Executor.Response response = executor.execute(Formulas.fromLispTree(LispTree.proto.parseFromString(formula)), null); LogInfo.logs("RESULT: %s", response.value); - checker.checkValues(((ListValue)response.value).values); + checker.checkValues(((ListValue) response.value).values); } SparqlExecutor executor = new SparqlExecutor(); @@ -76,6 +76,7 @@ public class SparqlExecutorTest { @Test(groups = "sparql") public void sparqlTrivial() { runFormula(executor, "fb:en.barack_obama", size(1)); + runFormula(executor, "(number 3)", size(1)); } @Test(groups = "sparql") public void sparqlJoin() { @@ -93,7 +94,7 @@ public class SparqlExecutorTest { @Test(groups = "sparql") public void sparqlLambda() { runFormula(executor, "((lambda x (!fb:people.person.parents (var x))) fb:en.barack_obama)", size(2)); // parents of Obama - runFormula(executor, "((lambda x (!fb:people.marriage.spouse (!fb:people.person.spouse_s (var x)))) fb:en.barack_obama)", size(1)); // spouse of Obama (only includes Michelle) + runFormula(executor, "((lambda x (!fb:people.marriage.spouse (!fb:people.person.spouse_s (var x)))) fb:en.barack_obama)", size(2)); // spouse of Barack Obama (includes himself) } @Test(groups = "sparql") public void sparqlReverse() { @@ -104,6 +105,10 @@ public class SparqlExecutorTest { @Test(groups = "sparql") public void sparqlUnion() { runFormula(executor, "(or (!fb:people.person.children fb:en.barack_obama) (!fb:people.person.parents fb:en.barack_obama))", size(4)); // children or parents of Obama + runFormula(executor, "(or fb:en.barack_obama fb:en.michelle_obama)", size(2)); // Barack and Michelle Obama + + // TODO(pliang): doesn't work + // runFormula(executor, "(or (number 3) (number 5))", size(2)); // 3 or 5 } @Test(groups = "sparql") public void sparqlNot() { @@ -125,6 +130,7 @@ public class SparqlExecutorTest { @Test(groups = "sparql") public void sparqlMark() { runFormula(executor, "(mark x (and (fb:type.object.type fb:people.person) (fb:people.person.place_of_birth (!fb:people.deceased_person.place_of_death (var x)))))", sizeAtLeast(10)); // people who were born in the place that they died + runFormula(executor, "((lambda x (mark y (!fb:people.marriage.spouse (!fb:people.person.spouse_s (and (var x) (!= (var y))))))) fb:en.barack_obama)", size(1)); // spouse of Barack Obama } @Test(groups = "sparql") public void sparqlAggregate() { @@ -134,8 +140,11 @@ public class SparqlExecutorTest { runFormula(executor, "(sum (!fb:location.location.area (fb:type.object.type fb:location.us_state)))", regexMatches("\\(number .* fb:en.square_kilometer\\)")); // total area of all US states runFormula(executor, "((reverse (lambda x (count (!fb:people.person.children (var x))))) (>= 50))", size(2)); // people with at least 50 children - runFormula(executor, "(and (fb:type.object.type fb:location.us_state) ((reverse (lambda x (count ((reverse (lambda y (fb:location.location.adjoin_s (fb:location.adjoining_relationship.adjoins (and (fb:type.object.type fb:location.us_state) (var y)))))) (var x))))) (mark x (> 6))))", size(4)); // states bordering more than 6 states - // TODO: known bug (bordering less than 2 states doesn't include Alaska and Hawaii - need to make things optional) + String border = "(lambda x (mark y (fb:location.location.adjoin_s (fb:location.adjoining_relationship.adjoins (and (var x) (!= (var y)))))))"; + runFormula(executor, "(and (fb:type.object.type fb:location.us_state) ((reverse (lambda x (count (and (fb:type.object.type fb:location.us_state) (" + border + " (var x)))))) (> 6)))", size(4)); // states bordering more than 6 states + + // TODO(pliang): known bug (bordering less than 2 states doesn't include Alaska and Hawaii - need to make things optional) + // (execute (and (@type @state) ((reverse (lambda x (count (and (@type @state) (@border (var x)))))) (number 0)))) # doesn't work } @Test(groups = "sparql") public void sparqlSuperlative() { @@ -145,10 +154,18 @@ public class SparqlExecutorTest { runFormula(executor, "(!fb:measurement_unit.dated_integer.number (argmax 1 1 (!fb:location.statistical_region.population fb:en.california) fb:measurement_unit.dated_integer.year))", size(1)); // (most recent) population of california runFormula(executor, "(!fb:measurement_unit.dated_integer.number (argmax 1 2 (!fb:location.statistical_region.population fb:en.california) fb:measurement_unit.dated_integer.year))", size(2)); // (most recent) two populations of california - - // TODO: this query seems to time out, figure out why. - //runFormula(executor, "(argmax 1 1 (fb:type.object.type fb:location.us_state) (lambda x (!fb:measurement_unit.dated_integer.number (argmax 1 1 (!fb:location.statistical_region.population (var x)) fb:measurement_unit.dated_integer.year))))", matches("(name fb:en.california)")); + + // TODO(pliang): this query seems to time out, figure out why. + // runFormula(executor, "(argmax 1 1 (fb:type.object.type fb:location.us_state) (lambda x (!fb:measurement_unit.dated_integer.number (argmax 1 1 (!fb:location.statistical_region.population (var x)) fb:measurement_unit.dated_integer.year))))", matches("(name fb:en.california)")); runFormula(executor, "(argmax 1 1 (fb:type.object.type fb:location.us_state) (reverse (lambda x (count ((reverse (lambda y (fb:location.location.adjoin_s (fb:location.adjoining_relationship.adjoins (and (fb:type.object.type fb:location.us_state) (var y)))))) (var x))))))", size(2)); // states that borders the most states } + + // Arithmetic + @Test(groups = "sparql") public void sparqlArithmetic() { + runFormula(executor, "(+ (!fb:people.person.height_meters fb:en.barack_obama) (!fb:people.person.height_meters fb:en.michelle_obama))", regexMatches("\\(number 3.650 fb:en.meter\\)")); + runFormula(executor, "(- (!fb:people.person.height_meters fb:en.barack_obama) (!fb:people.person.height_meters fb:en.michelle_obama))", regexMatches("\\(number 0.050 fb:en.meter\\)")); + runFormula(executor, "(/ (!fb:people.person.height_meters fb:en.barack_obama) (!fb:people.person.height_meters fb:en.michelle_obama))", regexMatches("\\(number 1.028\\)")); + runFormula(executor, "(+ (number 3) (number 8))", regexMatches("\\(number 11\\)")); + } } diff --git a/src/edu/stanford/nlp/sempre/test/StemmerTest.java b/src/edu/stanford/nlp/sempre/freebase/test/StemmerTest.java similarity index 78% rename from src/edu/stanford/nlp/sempre/test/StemmerTest.java rename to src/edu/stanford/nlp/sempre/freebase/test/StemmerTest.java index ca41014..12f9036 100644 --- a/src/edu/stanford/nlp/sempre/test/StemmerTest.java +++ b/src/edu/stanford/nlp/sempre/freebase/test/StemmerTest.java @@ -1,6 +1,6 @@ -package edu.stanford.nlp.sempre.test; +package edu.stanford.nlp.sempre.freebase.test; -import edu.stanford.nlp.sempre.Stemmer; +import edu.stanford.nlp.sempre.freebase.Stemmer; import org.testng.annotations.Test; import static org.testng.AssertJUnit.assertEquals; diff --git a/src/edu/stanford/nlp/sempre/test/TokenMatchTest.java b/src/edu/stanford/nlp/sempre/freebase/test/TokenMatchTest.java similarity index 83% rename from src/edu/stanford/nlp/sempre/test/TokenMatchTest.java rename to src/edu/stanford/nlp/sempre/freebase/test/TokenMatchTest.java index ccae2ba..fb77caf 100644 --- a/src/edu/stanford/nlp/sempre/test/TokenMatchTest.java +++ b/src/edu/stanford/nlp/sempre/freebase/test/TokenMatchTest.java @@ -1,6 +1,6 @@ -package edu.stanford.nlp.sempre.test; +package edu.stanford.nlp.sempre.freebase.test; -import edu.stanford.nlp.sempre.fbalignment.lexicons.TokenLevelMatchFeatures; +import edu.stanford.nlp.sempre.freebase.lexicons.TokenLevelMatchFeatures; import edu.stanford.nlp.stats.Counter; import org.testng.annotations.Test; diff --git a/src/edu/stanford/nlp/sempre/freebase/utils/CollectionUtils.java b/src/edu/stanford/nlp/sempre/freebase/utils/CollectionUtils.java new file mode 100644 index 0000000..f9788a0 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/utils/CollectionUtils.java @@ -0,0 +1,25 @@ +package edu.stanford.nlp.sempre.freebase.utils; + +import java.util.HashMap; +import java.util.Map; + +public final class CollectionUtils { + private CollectionUtils() { } + + public static Map arraysToMap(K[] keys, V[] values) { + if (keys.length != values.length) + throw new RuntimeException("Lenght of keys: " + keys.length + ", length of values: " + values.length); + Map res = new HashMap<>(); + for (int i = 0; i < keys.length; ++i) { + res.put(keys[i], values[i]); + } + return res; + } + + public static Map doubleContainerToDoubleMap(Map map) { + Map res = new HashMap<>(); + for (K key : map.keySet()) + res.put(key, map.get(key).value()); + return res; + } +} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/utils/DoubleContainer.java b/src/edu/stanford/nlp/sempre/freebase/utils/DoubleContainer.java similarity index 93% rename from src/edu/stanford/nlp/sempre/fbalignment/utils/DoubleContainer.java rename to src/edu/stanford/nlp/sempre/freebase/utils/DoubleContainer.java index 73ce700..0a0b231 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/utils/DoubleContainer.java +++ b/src/edu/stanford/nlp/sempre/freebase/utils/DoubleContainer.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.utils; +package edu.stanford.nlp.sempre.freebase.utils; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -18,7 +18,7 @@ public class DoubleContainer implements Comparable { public void inc(double n) { value += n; } public void dec(double n) { value -= n; } - + public void set(double n) { value = n; } public double value() { return value; } diff --git a/src/edu/stanford/nlp/sempre/fbalignment/utils/FileUtils.java b/src/edu/stanford/nlp/sempre/freebase/utils/FileUtils.java similarity index 93% rename from src/edu/stanford/nlp/sempre/fbalignment/utils/FileUtils.java rename to src/edu/stanford/nlp/sempre/freebase/utils/FileUtils.java index 64fdc20..40bf435 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/utils/FileUtils.java +++ b/src/edu/stanford/nlp/sempre/freebase/utils/FileUtils.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.utils; +package edu.stanford.nlp.sempre.freebase.utils; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -18,7 +18,8 @@ import java.util.*; * Utilities for files * @author jonathanberant */ -public class FileUtils { +public final class FileUtils { + private FileUtils() { } /** * Upload a set of string where each line is an element @@ -47,8 +48,8 @@ public class FileUtils { String[] tokens = line.split("\t"); res.put(tokens[0], tokens[1]); i++; - if(i % 1000000 == 0) - LogInfo.logs("Uploaing line %s: %s",i,line); + if (i % 1000000 == 0) + LogInfo.logs("Uploaing line %s: %s", i, line); } reader.close(); return res; @@ -121,7 +122,7 @@ public class FileUtils { } - public static BiMap loadString2IntegerBiMap(String file, String delimiter) throws IOException, NumberFormatException { + public static BiMap loadString2IntegerBiMap(String file, String delimiter) throws IOException { BiMap res = HashBiMap.create(); BufferedReader reader = IOUtils.getBufferedFileReader(file); @@ -136,7 +137,7 @@ public class FileUtils { return res; } - public static BiMap loadIntegerToIntegerBiMap(String file) throws IOException, NumberFormatException { + public static BiMap loadIntegerToIntegerBiMap(String file) throws IOException { BiMap res = HashBiMap.create(); BufferedReader reader = IOUtils.getBufferedFileReader(file); @@ -151,7 +152,7 @@ public class FileUtils { return res; } - public static Map loadString2IntegerMap(String file) throws IOException, NumberFormatException { + public static Map loadString2IntegerMap(String file) throws IOException { Map res = new HashMap(); BufferedReader reader = IOUtils.getBufferedFileReader(file); @@ -166,7 +167,7 @@ public class FileUtils { return res; } - public static BiMap loadString2IntegerBiMap(String file) throws IOException, NumberFormatException { + public static BiMap loadString2IntegerBiMap(String file) throws IOException { return loadString2IntegerBiMap(file, "\t"); } diff --git a/src/edu/stanford/nlp/sempre/freebase/utils/FormatConverter.java b/src/edu/stanford/nlp/sempre/freebase/utils/FormatConverter.java new file mode 100644 index 0000000..908bdff --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/utils/FormatConverter.java @@ -0,0 +1,82 @@ +package edu.stanford.nlp.sempre.freebase.utils; + +public final class FormatConverter { + private FormatConverter() { } + + public static final String FULL_FB_PREFIX = "http://rdf.freebase.com/ns/"; + public static final String SHORT_FB_PREFIX = "fb:"; + + public static String fromDotToSlash(String dotString) { + if (dotString.startsWith("http")) { + return dotString.substring(dotString.lastIndexOf('/')).replace('.', '/'); + } else { + if (dotString.startsWith("/")) + return dotString; + return "/" + dotString.substring(dotString.indexOf(':') + 1).replace('.', '/'); + } + } + + public static String toShortPrefix(String str) { + return str.replace(FULL_FB_PREFIX, SHORT_FB_PREFIX); + } + + public static String fromNoPrefixMidToDot(String mid) { + + if (mid.startsWith("fb:m") || mid.startsWith("/m/")) + throw new RuntimeException("This mid has a prefix: " + mid); + return SHORT_FB_PREFIX + "m." + mid; + + } + + /** converts from slash notation to dot notation */ + public static String fromSlashToDot(String slashString, boolean strict) { + + if (!(slashString.charAt(0) == '/')) { + if (strict) { + throw new IllegalArgumentException("Not a legal slash string: " + slashString); + } else + return slashString; + } + + return SHORT_FB_PREFIX + slashString.substring(1).replace('/', '.'); + } + + /** + * convert from [/award/award_winning_work/awards_won, + * /award/award_honor/award_winner] to (lambda x + * (fb:award.award_winning_work.awards_won (fb:award.award_honor.award_winner + * (var x)))) + */ + public static String fromCvtBinaryToLispTree(String str) { + + boolean reversed = false; + if (str.startsWith("!")) { + reversed = true; + str = str.substring(1); + } + // strip brackets + str = str.substring(1, str.length() - 1); + String[] tokens = str.split(","); + if (tokens.length == 1) { + return reversed ? "!" + fromSlashToDot(tokens[0].trim(), false) : fromSlashToDot(tokens[0].trim(), false); + } else { + String property1 = fromSlashToDot(tokens[0].trim(), false); + String property2 = fromSlashToDot(tokens[1].trim(), false); + return propertiesToCompositeLispTree(property1, property2, reversed); + } + } + + public static String propertiesToCompositeLispTree(String property1, + String property2, boolean reversed) { + + StringBuilder sb = new StringBuilder(); + if (reversed) { + sb.append("(lambda x (!" + property2 + " (!" + property1 + " (var x))))"); + } else { + sb.append("(lambda x (" + property1 + " (" + property2 + " (var x))))"); + } + return sb.toString(); + } + + +} diff --git a/src/edu/stanford/nlp/sempre/freebase/utils/FreebaseUtils.java b/src/edu/stanford/nlp/sempre/freebase/utils/FreebaseUtils.java new file mode 100644 index 0000000..1563538 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/utils/FreebaseUtils.java @@ -0,0 +1,180 @@ +package edu.stanford.nlp.sempre.freebase.utils; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import edu.stanford.nlp.io.IOUtils; +import fig.basic.LogInfo; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + + +/** + * Utils for handling freebase data dump files + * + * @author jonathanberant + */ +public final class FreebaseUtils { + private FreebaseUtils() { } + + public static final Pattern DELIMITER_PATTERN = Pattern.compile("\t"); + public static final String NAME_PROPERTY = "/type/object/name"; + public static final String ALIAS_PROPERTY = "/common/topic/alias"; + public static final String TYPE_PROPERTY = "/type/object/type"; + public static final String PROFESSION_PROPERTY = "/people/person/profession"; + public static final String MID_PREFIX = "/m/"; + public static final String COMMON_DOMAIN_PREFIX = "/common/"; + public static final String USER_DOMAIN_PREFIX = "/user/"; + public static final String BASE_DOMAIN_PREFIX = "/base/"; + public static final String FREEBASE_DOMAIN_PREFIX = "/freebase/"; + public static final String DATA_DOMAIN_PREFIX = "/dataworld/"; + public static final String TYPE_DOMAIN_PREFIX = "/type/"; + // indices in data dump file + public static final int MID_INDEX = 0; + public static final int PROPERTY_INDEX = 1; + public static final int LANGUAGE_INDEX = 2; + public static final int VALUE_INDEX = 2; + public static final int DATE_INDEX = 3; + public static final int NAME_INDEX = 3; + + /** Checks if a string is a valid MID */ + public static boolean isMid(String str) { + return str.startsWith("/m/"); + } + + public static boolean isValidPropertyLine(String line) { + + String[] tokens = DELIMITER_PATTERN.split(line); + + return tokens.length == 3 && + isMid(tokens[MID_INDEX]) && + isMid(tokens[VALUE_INDEX]) && + !isMid(tokens[PROPERTY_INDEX]) && + isValidTypePrefix(tokens[PROPERTY_INDEX]); + } + + public static boolean isValidPropertyLineWithDate(String line) { + + String[] tokens = DELIMITER_PATTERN.split(line); + + boolean regularProperty = tokens.length == 3 && + isMid(tokens[MID_INDEX]) && + !isMid(tokens[PROPERTY_INDEX]) && + isValidTypePrefix(tokens[PROPERTY_INDEX]); + + boolean dateProperty = + tokens.length == 4 && + isMid(tokens[MID_INDEX]) && + isValidTypePrefix(tokens[PROPERTY_INDEX]) && + tokens[2].equals("") && + isDate(tokens[3]); + + return regularProperty || dateProperty; + } + + public static boolean isDate(String dateCandidate) { + + boolean res = true; + if (dateCandidate.startsWith("-")) { + dateCandidate = dateCandidate.substring(1); + } + int i = 0; + for (; i < Math.min(4, dateCandidate.length()); ++i) { + if (!Character.isDigit(dateCandidate.charAt(i))) { + res = false; + break; + } + } + if (i != 4) + res = false; + if (dateCandidate.length() > 4 && dateCandidate.charAt(4) != '-') + res = false; + return res; + } + + public static String extractDate(String dateCandidate) { + + boolean neg = false; + if (dateCandidate.startsWith("-")) { + neg = true; + dateCandidate = dateCandidate.substring(1); + } + return neg ? "-" + dateCandidate.substring(0, 4) : dateCandidate.substring(0, 4); + } + + public static boolean isValidTypePrefix(String type) { + if (type.equals("/type/datetime")) + return true; + return !(type.startsWith(BASE_DOMAIN_PREFIX) || type.startsWith(MID_PREFIX) || type.startsWith(COMMON_DOMAIN_PREFIX) || + type.startsWith(USER_DOMAIN_PREFIX) || + type.startsWith(DATA_DOMAIN_PREFIX) || + type.startsWith(FREEBASE_DOMAIN_PREFIX) || + type.startsWith(TYPE_DOMAIN_PREFIX) || + type.startsWith("/guid/")); + } + + public static String getNoPrefixMid(String line) { + return DELIMITER_PATTERN.split(line)[MID_INDEX].substring(MID_PREFIX.length()); + } + + public static String getProperty(String line) { + return DELIMITER_PATTERN.split(line)[PROPERTY_INDEX]; + } + + public static String getValue(String line) { + return DELIMITER_PATTERN.split(line)[VALUE_INDEX]; + } + + public static String getDateValue(String line) { + return DELIMITER_PATTERN.split(line)[DATE_INDEX]; + } + + public static String getName(String line) { + return DELIMITER_PATTERN.split(line)[NAME_INDEX]; + } + + public static boolean isArg1Equal2Mid(String mid, String tupleTokens) { + return DELIMITER_PATTERN.split(tupleTokens)[MID_INDEX].equals(mid); + } + + public static Map loadMid2NameMap(String filename) throws IOException { + + LogInfo.log("Loading mid to name file..."); + + Map res = new HashMap(); + BufferedReader reader = IOUtils.getBufferedFileReader(filename); + String line; + while ((line = reader.readLine()) != null) { + String[] tokens = line.split("\t"); + res.put(tokens[0], tokens[1]); + } + LogInfo.log("Loaded " + res.keySet().size() + " MIDs"); + return res; + } + + public static BiMap loadProperties(String propertyFileName) throws IOException { + + BiMap res = HashBiMap.create(); + BufferedReader reader = IOUtils.getBufferedFileReader(propertyFileName); + + String line; + short id = 1; + while ((line = reader.readLine()) != null) { + res.put(id++, line); + } + return res; + } + + public static boolean isUnary(String property) { + property = FormatConverter.fromDotToSlash(property); + return (property.equals(TYPE_PROPERTY) || property.equals(PROFESSION_PROPERTY)); + } + + public static boolean isNameProperty(String property) { + property = FormatConverter.fromDotToSlash(property); + return (property.equals(ALIAS_PROPERTY) || property.equals(NAME_PROPERTY)); + } +} diff --git a/src/edu/stanford/nlp/sempre/freebase/utils/LinkedExtractionFileUtils.java b/src/edu/stanford/nlp/sempre/freebase/utils/LinkedExtractionFileUtils.java new file mode 100644 index 0000000..fcca557 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/utils/LinkedExtractionFileUtils.java @@ -0,0 +1,110 @@ +package edu.stanford.nlp.sempre.freebase.utils; + +import edu.stanford.nlp.io.IOUtils; +import fig.basic.LogInfo; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.*; +import java.util.regex.Pattern; + +public final class LinkedExtractionFileUtils { + + private String extractionFile; + private static final int ARG1_INDEX = 0; + private static final int PREDICATE_INDEX = 1; + private static final int ARG2_INDEX = 2; + private static final int MID_INDEX = 3; + public static final Pattern DELIMITER_PATTERN = Pattern.compile("\t"); + public static final String TIME_ARG = "TIME:"; + + + public LinkedExtractionFileUtils(String extractionFileName) { + this.extractionFile = extractionFileName; + } + + public Map> getIdToExtractionsMap() throws IOException { + + LogInfo.log("Uploading id-to-extraction-set map"); + Map> res = new HashMap>(); + + BufferedReader reader = IOUtils.getBufferedFileReader(extractionFile); + String line; + while ((line = reader.readLine()) != null) { + + String[] tokens = DELIMITER_PATTERN.split(line); + + Set extractionSet = res.get(tokens[MID_INDEX]); + if (extractionSet == null) { + extractionSet = new HashSet(); + res.put(tokens[MID_INDEX], extractionSet); + } + extractionSet.add(tokens[ARG1_INDEX] + DELIMITER_PATTERN + tokens[PREDICATE_INDEX] + DELIMITER_PATTERN + DELIMITER_PATTERN + tokens[ARG2_INDEX]); + } + reader.close(); + LogInfo.log("Done uploading id-to-extraction-set map"); + return res; + } + + public Set getLinkedIdSet() throws IOException { + + LogInfo.log("Uploading linked MIDs set"); + Set res = new HashSet(); + + BufferedReader reader = IOUtils.getBufferedFileReader(extractionFile); + String line; + while ((line = reader.readLine()) != null) { + + String[] tokens = DELIMITER_PATTERN.split(line); + res.add(tokens[MID_INDEX]); + } + reader.close(); + LogInfo.log("Done uploading linked IDs set"); + return res; + } + + public Map>> getIdToArg2ToPredicateListMap() throws IOException { + + LogInfo.begin_track("Uploading id-to-arg-predicate-list-map"); + // BinaryNormalizer normalizer = new BinaryNormalizer(); + + Map>> res = new HashMap>>(); + + for (String line : IOUtils.readLines(extractionFile)) { + + String[] tokens = DELIMITER_PATTERN.split(line); + + String id = tokens[MID_INDEX]; + String arg2 = tokens[ARG2_INDEX]; + String predicate = tokens[PREDICATE_INDEX]; + // String predicate = normalizer.normalize(tokens[PREDICATE_INDEX]); + + Map> arg2ToPredicateList = res.get(id); + if (arg2ToPredicateList == null) { + arg2ToPredicateList = new HashMap>(); + arg2ToPredicateList.put(arg2, new LinkedList()); + res.put(id, arg2ToPredicateList); + } + + List predicateList = arg2ToPredicateList.get(arg2); + if (predicateList == null) { + predicateList = new LinkedList(); + arg2ToPredicateList.put(arg2, predicateList); + } + predicateList.add(predicate); + } + LogInfo.end_track(); + return res; + } + + public static boolean isTimeArg(String str) { + return str.startsWith(TIME_ARG); + } + + public static String extractTime(String str) { + if (!isTimeArg(str)) { + throw new RuntimeException("Not a time arg: " + str); + } + return str.substring(TIME_ARG.length()); + } +} diff --git a/src/edu/stanford/nlp/sempre/fbalignment/utils/MathUtils.java b/src/edu/stanford/nlp/sempre/freebase/utils/MathUtils.java similarity index 74% rename from src/edu/stanford/nlp/sempre/fbalignment/utils/MathUtils.java rename to src/edu/stanford/nlp/sempre/freebase/utils/MathUtils.java index 92e99d6..e5761fa 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/utils/MathUtils.java +++ b/src/edu/stanford/nlp/sempre/freebase/utils/MathUtils.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.utils; +package edu.stanford.nlp.sempre.freebase.utils; import edu.stanford.nlp.stats.ClassicCounter; import edu.stanford.nlp.stats.Counter; @@ -9,7 +9,8 @@ import java.util.List; import java.util.Map; import java.util.Set; -public class MathUtils { +public final class MathUtils { + private MathUtils() { } public static double jaccard(double intersection, double size1, double size2, double smoothing) { return intersection / (size1 + size2 + smoothing - intersection); @@ -77,24 +78,24 @@ public class MathUtils { } return Math.sqrt(sqDistance); } - - public static double sumDoubleMap(Map map) { + + public static double sumDoubleMap(Map map) { double sum = 0.0; - for(DoubleContainer d: map.values()) - sum+=d.value(); + for (DoubleContainer d : map.values()) + sum += d.value(); return sum; } - - public static void normalizeDoubleMap(Map map) { + + public static void normalizeDoubleMap(Map map) { double sum = 0.0; - for(DoubleContainer d: map.values()) - sum+=d.value(); - for(T key: map.keySet()) { + for (DoubleContainer d : map.values()) + sum += d.value(); + for (T key : map.keySet()) { double normalizedValue = map.get(key).value() / sum; map.get(key).set(normalizedValue); } } - + /** * Computes jaccard between sets of objects * @param x @@ -102,16 +103,16 @@ public class MathUtils { * @return */ public static double jaccard(Set x, Set y) { - + Set intersection = new HashSet(x); intersection.retainAll(y); - Set union= new HashSet(x); + Set union = new HashSet(x); union.addAll(y); - + double res = union.size() == 0 ? 1.0 : (double) intersection.size() / union.size(); return res; } - + /** * Computes jaccard between sets of objects * @param x @@ -119,16 +120,16 @@ public class MathUtils { * @return */ public static double jaccard(List x, List y) { - + Set intersection = new HashSet(x); intersection.retainAll(y); - Set union= new HashSet(x); + Set union = new HashSet(x); union.addAll(y); - + double res = union.size() == 0 ? 1.0 : (double) intersection.size() / union.size(); return res; } - + /** * how many of the tokens in x are covered by y * @param x @@ -137,65 +138,65 @@ public class MathUtils { */ public static double coverage(List x, List y) { Set yTokens = new HashSet(y); - int covered=0; - for(T xItem: x) { - if(yTokens.contains(xItem)) + int covered = 0; + for (T xItem : x) { + if (yTokens.contains(xItem)) covered++; } return (double) covered / x.size(); } - + /** * Geometric average of unigram bigram and trigram precision * @param test * @param ref * @return */ - + public static double bleu(List test, List ref) { - + Set refUnigrams = new HashSet(); Set refBigrams = new HashSet(); Set refTrigrams = new HashSet(); - for(int i = 0; i < ref.size(); ++i) { + for (int i = 0; i < ref.size(); ++i) { refUnigrams.add(ref.get(i)); - if(i < ref.size()-1) - refBigrams.add(ref.get(i)+" "+ref.get(i+1)); - if(i < ref.size()-2) - refTrigrams.add(ref.get(i)+" "+ref.get(i+1)+" "+ref.get(i+2)); + if (i < ref.size() - 1) + refBigrams.add(ref.get(i) + " " + ref.get(i + 1)); + if (i < ref.size() - 2) + refTrigrams.add(ref.get(i) + " " + ref.get(i + 1) + " " + ref.get(i + 2)); } - int unigramCov=0; - int bigramCov=0; - int trigramCov=0; - for(int i = 0; i < test.size(); ++i) { - if(refUnigrams.contains(test.get(i))) + int unigramCov = 0; + int bigramCov = 0; + int trigramCov = 0; + for (int i = 0; i < test.size(); ++i) { + if (refUnigrams.contains(test.get(i))) unigramCov++; - if(i < test.size()-1) { - String bigram = test.get(i)+" "+test.get(i+1); - if(refBigrams.contains(bigram)) + if (i < test.size() - 1) { + String bigram = test.get(i) + " " + test.get(i + 1); + if (refBigrams.contains(bigram)) bigramCov++; } - if(i < test.size()-2) { - String trigram = test.get(i)+" "+test.get(i+1)+" "+test.get(i+2); - if(refTrigrams.contains(trigram)) + if (i < test.size() - 2) { + String trigram = test.get(i) + " " + test.get(i + 1) + " " + test.get(i + 2); + if (refTrigrams.contains(trigram)) trigramCov++; - } + } } double unigramPrec = (double) unigramCov / test.size(); - double bigramPrec = (double) bigramCov / (test.size()-1); - double trigramPrec = (double) trigramCov / (test.size()-2); + double bigramPrec = (double) bigramCov / (test.size() - 1); + double trigramPrec = (double) trigramCov / (test.size() - 2); double exponent = (double) 1 / 3; - return Math.pow(unigramPrec*bigramPrec*trigramPrec,exponent); + return Math.pow(unigramPrec * bigramPrec * trigramPrec, exponent); } - + public static double tokensCosine(List x, List y) { - + Counter xCounter = new ClassicCounter(); - for(String str: x) + for (String str : x) xCounter.incrementCount(str); Counter yCounter = new ClassicCounter(); - for(String str: y) + for (String str : y) yCounter.incrementCount(str); - return Counters.cosine(xCounter, yCounter); + return Counters.cosine(xCounter, yCounter); } } diff --git a/src/edu/stanford/nlp/sempre/freebase/utils/SemparseLogTools.java b/src/edu/stanford/nlp/sempre/freebase/utils/SemparseLogTools.java new file mode 100644 index 0000000..01fd1be --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/utils/SemparseLogTools.java @@ -0,0 +1,261 @@ +package edu.stanford.nlp.sempre.freebase.utils; + +import edu.stanford.nlp.io.IOUtils; +import fig.basic.LispTree; +import fig.basic.LogInfo; +import fig.basic.MapUtils; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +public final class SemparseLogTools { + private SemparseLogTools() { } + + public static void main(String[] args) throws IOException { + + LogInfo.begin_track_printAll("analyze"); + if (args[0].equals("compare")) { + compareLogs(args[1], args[2], args[3]); + } else if (args[0].equals("diff")) { + printDiff(args[1], args[2]); + } else if (args[0].equals("fb_descriptions")) { + getFbDescInTrueDerivations(args[1], args[2]); + } + if (args[0].equals("result_list")) { + printResultList(args[1], args[2]); + } + LogInfo.end_track(); + } + + private static void printResultList(String log, String field) { + + boolean start = false; + int numOfIterations = -1; + String prob = null; + String correct = null; + String oracle = null; + + + for (String line : IOUtils.readLines(log)) { + if (line.contains("Iteration")) { + int slashIndex = line.indexOf('/'); + int openCurlyIndex = line.indexOf('{'); + numOfIterations = Integer.parseInt(line.substring(slashIndex + 1, openCurlyIndex - 1)); + } + if (line.contains("Processing iter=" + numOfIterations + ".dev")) { + start = true; + } + if (start) { + if (line.contains("Pred@0000")) { + prob = line.substring(line.indexOf("prob=") + 5, line.indexOf(", comp=")); + } + if (line.contains("Example:")) + prob = "0"; + if (line.contains("Current: correct=")) { + int correctIndex = line.indexOf("correct="); + int oracleIndex = line.indexOf("oracle="); + int partCorrectIndex = line.indexOf("partCorrect="); + int parsedIndex = line.indexOf("parsed="); + int numTokensIndex = line.indexOf("numTokens="); + + + if (field.equals("oracle")) + System.out.println(line.substring(oracleIndex + 7, partCorrectIndex - 1)); + if (field.equals("correct")) + System.out.println(line.substring(correctIndex + 8, oracleIndex - 1) + "\t" + prob); + if (field.equals("parsed")) + System.out.println(line.substring(parsedIndex + 7, numTokensIndex - 1)); + } + } + } + } + + private static void getFbDescInTrueDerivations(String log, String out) throws IOException { + + boolean start = false; + int numOfIterations = -1; + Map> exampleToDescriptions = new HashMap>(); + + + String currExample = null; + + for (String line : IOUtils.readLines(log)) { + if (line.contains("Iteration")) { + int slashIndex = line.indexOf('/'); + int openCurlyIndex = line.indexOf('{'); + numOfIterations = Integer.parseInt(line.substring(slashIndex + 1, openCurlyIndex - 1)); + } + if (line.contains("Processing iter=" + numOfIterations + ".")) { + start = true; + } + if (start) { + if (line.contains("Example:")) { + int end = line.indexOf("{") - 1; + currExample = line.substring(line.indexOf("Example:") + 9, end); + exampleToDescriptions.put(currExample, new HashSet()); + } + if (line.contains("True@")) { + + String formula = line.substring(line.indexOf("(formula"), line.indexOf("(value") - 1); + LispTree t = LispTree.proto.parseFromString(formula); + Set descriptions = new HashSet(); + extractDescriptionsFromTree(t, descriptions); + for (String description : descriptions) + MapUtils.addToSet(exampleToDescriptions, currExample, description); + } + } + } + PrintWriter writer = IOUtils.getPrintWriter(out); + for (String example : exampleToDescriptions.keySet()) { + if (exampleToDescriptions.get(example).size() > 0) + writer.println(example + "\t" + exampleToDescriptions.get(example)); + } + writer.close(); + } + + private static void extractDescriptionsFromTree(LispTree t, Set descriptions) { + + if (t.value != null) { + if (t.value.indexOf('.') != t.value.lastIndexOf('.')) { + descriptions.add(t.value.substring(t.value.lastIndexOf('.') + 1)); + } + } + if (!t.isLeaf()) { + for (LispTree child : t.children) { + extractDescriptionsFromTree(child, descriptions); + } + } + } + + private static void printDiff(String log, String field) throws IOException { + + boolean start = false; + int numOfIterations = -1; + + String example = null; + String targetFormula = null; + String targetValue = null; + String trueDeriv = null; + String predDeriv = null; + + boolean print = false; + + for (String line : IOUtils.readLines(log)) { + if (line.contains("Iteration")) { + int slashIndex = line.indexOf('/'); + int openCurlyIndex = line.indexOf('{'); + numOfIterations = Integer.parseInt(line.substring(slashIndex + 1, openCurlyIndex - 1)); + } + if (line.contains("Processing iter=" + numOfIterations + ".dev")) { + start = true; + } + if (start) { + if (line.contains("Example:")) { + if (print && example != null) { + LogInfo.log(example); + LogInfo.log(targetFormula); + LogInfo.log(targetValue); + LogInfo.log(trueDeriv); + LogInfo.log(predDeriv); + } + example = line; + targetFormula = null; targetValue = null; trueDeriv = null; + predDeriv = null; + } + if (line.contains("targetFormula:")) { + targetFormula = line; + } + if (line.contains("targetValue:")) { + targetValue = line; + } + if (line.contains("True@") && trueDeriv == null) { + trueDeriv = line; + } + if (line.contains("Pred@") && predDeriv == null) { + predDeriv = line; + } + if (line.contains("Current:")) { + if (field.equals("correct")) { + if (line.contains("correct=0") && line.contains("oracle=1")) { + print = true; + } else + print = false; + } + if (field.equals("oracle")) { + if (line.contains("oracle=0") && line.contains("parsed=1")) + print = true; + else + print = false; + } + if (field.equals("parsed")) { + if (line.contains("parsed=0")) + print = true; + else + print = false; + } + } + } + } + } + + public static void compareLogs(String log1, String log2, String field) { + List correctnessList1 = computeCorrectnessList(log1, field); + List correctnessList2 = computeCorrectnessList(log2, field); + + if (correctnessList1.size() != correctnessList2.size()) + throw new RuntimeException("lists are not same size"); + LogInfo.logs("Size of correctness: %s", correctnessList1.size()); + + for (int i = 0; i < correctnessList1.size(); ++i) { + if (!correctnessList1.get(i).equals(correctnessList2.get(i))) { + LogInfo.log("example: " + i + " log1: " + correctnessList1.get(i) + " log2: " + correctnessList2.get(i)); + } + } + } + + private static List computeCorrectnessList(String log1, String field) { + + List res = new LinkedList<>(); + boolean start = false; + int numOfIterations = -1; + for (String line : IOUtils.readLines(log1)) { + if (line.contains("Iteration")) { + int slashIndex = line.indexOf('/'); + int openCurlyIndex = line.indexOf('{'); + numOfIterations = Integer.parseInt(line.substring(slashIndex + 1, openCurlyIndex - 1)); + LogInfo.logs("Number of iterations=%s", numOfIterations); + } + if (line.contains("Processing iter=" + numOfIterations + ".dev")) { + start = true; + } + if (start) { + if (line.contains("Current: parsed=")) { + String[] tokens = line.split("\\s+"); + for (String token : tokens) { + String[] tokenParts = token.split("="); + if (field.equals(tokenParts[0])) + res.add(Double.parseDouble(tokenParts[1])); + } +// LogInfo.logs() +// int correctIndex = line.indexOf(" correct="); +// int oracleIndex = line.indexOf(" oracle="); +// int partCorrectIndex = line.indexOf(" partCorrect="); +// int partOracleIndex= line.indexOf(" partOracle="); +// int afterPartOracleIndex = line.indexOf("\\s+",partOracleIndex); +// LogInfo.logs("part oracle index=%s, after=%s",partCorrectIndex,afterPartOracleIndex); +// if (field.equals("oracle")) +// res.add(Double.parseDouble(line.substring(oracleIndex + 8, partCorrectIndex))); +// if (field.equals("correct")) +// res.add(Double.parseDouble(line.substring(correctIndex + 9, oracleIndex))); +// if (field.equals("partCorrect")) +// res.add(Double.parseDouble(line.substring(partCorrectIndex + 13, partOracleIndex))); +// if (field.equals("partOracle")) +// res.add(Double.parseDouble(line.substring(partOracleIndex + 12, afterPartOracleIndex))); + } + } + } + return res; + } + +} diff --git a/src/edu/stanford/nlp/sempre/freebase/utils/ShortContainer.java b/src/edu/stanford/nlp/sempre/freebase/utils/ShortContainer.java new file mode 100644 index 0000000..c7674c9 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/freebase/utils/ShortContainer.java @@ -0,0 +1,28 @@ +package edu.stanford.nlp.sempre.freebase.utils; + +import java.io.Serializable; + +public class ShortContainer implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -911790554283478225L; + + private short count; + + public ShortContainer(short count) { this.count = count; } + + public void inc() { count++; } + + public void dec() { count--; } + + public void inc(short n) { count += n; } + + public void dec(short n) { count -= n; } + + public short value() { return count; } + + public String toString() { return new Short(count).toString(); } +} + diff --git a/src/edu/stanford/nlp/sempre/fbalignment/utils/WnExpander.java b/src/edu/stanford/nlp/sempre/freebase/utils/WnExpander.java similarity index 93% rename from src/edu/stanford/nlp/sempre/fbalignment/utils/WnExpander.java rename to src/edu/stanford/nlp/sempre/freebase/utils/WnExpander.java index d1435d4..c4de6e6 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/utils/WnExpander.java +++ b/src/edu/stanford/nlp/sempre/freebase/utils/WnExpander.java @@ -1,9 +1,9 @@ -package edu.stanford.nlp.sempre.fbalignment.utils; +package edu.stanford.nlp.sempre.freebase.utils; import fig.basic.LogInfo; import fig.basic.Option; -import edu.stanford.nlp.sempre.fbalignment.utils.WordNet.*; +import edu.stanford.nlp.sempre.freebase.utils.WordNet.*; import java.io.File; import java.io.IOException; @@ -43,18 +43,20 @@ public class WnExpander { case "hyponym": edgeTypes.add(EdgeType.HYPONYM); break; + default: + throw new RuntimeException("Invalid relation: " + wnRelation); } } } public Set expandPhrase(String phrase) { - //find synsetse for phrase + // find synsetse for phrase Set phraseSynsets = phraseToSynsets(phrase); - //expand synsets + // expand synsets for (EdgeType edgeType : edgeTypes) phraseSynsets.addAll(expandSynsets(phraseSynsets, edgeType)); - //find phrases for synsets + // find phrases for synsets Set expansions = synsetsToPhrases(phraseSynsets); if (opts.verbose > 0) { for (String expansion : expansions) diff --git a/src/edu/stanford/nlp/sempre/fbalignment/utils/WordNet.java b/src/edu/stanford/nlp/sempre/freebase/utils/WordNet.java similarity index 85% rename from src/edu/stanford/nlp/sempre/fbalignment/utils/WordNet.java rename to src/edu/stanford/nlp/sempre/freebase/utils/WordNet.java index 3c75f2c..f5d1daf 100644 --- a/src/edu/stanford/nlp/sempre/fbalignment/utils/WordNet.java +++ b/src/edu/stanford/nlp/sempre/freebase/utils/WordNet.java @@ -1,4 +1,4 @@ -package edu.stanford.nlp.sempre.fbalignment.utils; +package edu.stanford.nlp.sempre.freebase.utils; /* * Distributed as part of WordWalk, a Java package for lexical @@ -46,7 +46,7 @@ import edu.stanford.nlp.objectbank.ObjectBank; * @author dramage * @author Chris Manning made minimal changes to make it JavaNLP land not Ramage land */ -public class WordNet { +public final class WordNet { /** Global counter for assigning unique index to each loaded id. */ private final ArrayList all = new ArrayList(500000); @@ -58,9 +58,9 @@ public class WordNet { private final Set loadedEdges = new HashSet(); /** Indexes mapping canonical strings to their typed versions */ - private final Map synsets = new HashMap(); - private final Map words = new HashMap(); - private final Map wordtags = new HashMap(); + private final Map synsets = new HashMap(); + private final Map words = new HashMap(); + private final Map wordtags = new HashMap(); /** File resource backing this WordNet instance */ private final File path; @@ -116,21 +116,22 @@ public class WordNet { /** Set of relations that are transposes of eachother. */ private static final EdgeType[][] transpose = { - { EdgeType.HYPONYM, EdgeType.HYPERNYM }, - { EdgeType.INSTANCE_OF, EdgeType.HAS_INSTANCE }, - { EdgeType.ENTAILS, EdgeType.ENTAILED_BY }, - { EdgeType.SIM_HEAD, EdgeType.SIM_SATELLITE }, - { EdgeType.MM_HOLONYM, EdgeType.MM_MERONYM }, - { EdgeType.MS_HOLONYM, EdgeType.MS_MERONYM }, - { EdgeType.MP_HOLONYM, EdgeType.MP_MERONYM }, - { EdgeType.CAUSED_BY, EdgeType.CAUSES }, - { EdgeType.DERIVES, EdgeType.DERIVED_FROM }, - { EdgeType.PARTICIPLE_OF, EdgeType.HAS_PARTICIPLE }, - { EdgeType.PERTAINS_TO, EdgeType.PERTANYM_OF }, - { EdgeType.SEE_ALSO_TO, EdgeType.SEE_ALSO_FROM }, - { EdgeType.TERM_HAS_TOPIC, EdgeType.TOPIC_FROM_TERM }, - { EdgeType.TERM_HAS_USAGE, EdgeType.USAGE_FROM_TERM }, - { EdgeType.TERM_IN_REGION, EdgeType.REGION_HAS_TERM } }; + {EdgeType.HYPONYM, EdgeType.HYPERNYM}, + {EdgeType.INSTANCE_OF, EdgeType.HAS_INSTANCE}, + {EdgeType.ENTAILS, EdgeType.ENTAILED_BY}, + {EdgeType.SIM_HEAD, EdgeType.SIM_SATELLITE}, + {EdgeType.MM_HOLONYM, EdgeType.MM_MERONYM}, + {EdgeType.MS_HOLONYM, EdgeType.MS_MERONYM}, + {EdgeType.MP_HOLONYM, EdgeType.MP_MERONYM}, + {EdgeType.CAUSED_BY, EdgeType.CAUSES}, + {EdgeType.DERIVES, EdgeType.DERIVED_FROM}, + {EdgeType.PARTICIPLE_OF, EdgeType.HAS_PARTICIPLE}, + {EdgeType.PERTAINS_TO, EdgeType.PERTANYM_OF}, + {EdgeType.SEE_ALSO_TO, EdgeType.SEE_ALSO_FROM}, + {EdgeType.TERM_HAS_TOPIC, EdgeType.TOPIC_FROM_TERM}, + {EdgeType.TERM_HAS_USAGE, EdgeType.USAGE_FROM_TERM}, + {EdgeType.TERM_IN_REGION, EdgeType.REGION_HAS_TERM} + }; /** Set of relations that are self-reflexive, i.e. if a->b then b-> a */ private static final EdgeType[] reflexive = { @@ -163,7 +164,7 @@ public class WordNet { if (ssType == 's') { return ADJECTIVE; } - throw new IllegalArgumentException("Unexpected ss_type: "+ssType); + throw new IllegalArgumentException("Unexpected ss_type: " + ssType); } } @@ -173,8 +174,8 @@ public class WordNet { private int index; /** Outgoing links from this node */ - private Map> links - = new EnumMap>(EdgeType.class); + private Map> links + = new EnumMap>(EdgeType.class); /** Private no-arg constructor prevents new subclasses */ private WordNetID() { @@ -232,7 +233,7 @@ public class WordNet { @Override public boolean equals(Object other) { if (other == this) return true; - if ( ! (other instanceof WordNetID)) return false; + if (!(other instanceof WordNetID)) return false; WordNetID w = (WordNetID) other; return this.index == w.index; } @@ -252,17 +253,17 @@ public class WordNet { * * @author dramage */ - public class WordID extends WordNetID { + public final class WordID extends WordNetID { /** WordTags for this Word */ - private final EnumMap mWordTags - = new EnumMap(PartOfSpeech.class); + private final EnumMap mWordTags + = new EnumMap(PartOfSpeech.class); /** The string of our word (with spaces converted to underscore) */ public final String word; /** Immutable view of all WordTagIDs this WordID is part of. */ - public final Map wordTags + public final Map wordTags = Collections.unmodifiableMap(mWordTags); private WordID(String word) { @@ -300,7 +301,7 @@ public class WordNet { * * @author dramage */ - public class WordTagID extends WordNetID { + public final class WordTagID extends WordNetID { /** Mutable view of word senses for this WordTag. */ private final ArrayList mSynsets @@ -316,7 +317,7 @@ public class WordNet { public final List synsets = Collections.unmodifiableList(mSynsets); - public List getSynsets(){ + public List getSynsets() { return this.synsets; } private WordTagID(WordID word, PartOfSpeech tag) { @@ -340,10 +341,10 @@ public class WordNet { /** Adds the given Synset into this WordTag with the given number */ private void addSynset(SynsetID synsetId, int num) { num--; // convert to 0-indexed - assert num>=0 && (mSynsets.size()<=num || - mSynsets.get(num)==null || mSynsets.get(num)==synsetId) - : "Unexpected repeat of word sense "+synsetId+" on "+this - +": already "+mSynsets.get(num); + assert num >= 0 && (mSynsets.size() <= num || + mSynsets.get(num) == null || mSynsets.get(num) == synsetId) + : "Unexpected repeat of word sense " + synsetId + " on " + this + + ": already " + mSynsets.get(num); while (mSynsets.size() <= num) { mSynsets.add(null); @@ -353,7 +354,7 @@ public class WordNet { @Override public String toString() { - return word+"#"+tag.ssType; + return word + "#" + tag.ssType; } } @@ -367,7 +368,7 @@ public class WordNet { * * @author dramage */ - public class SynsetID extends WordNetID { + public final class SynsetID extends WordNetID { /** Modifiable WordTagIDs associated with this SynsetID */ private final ArrayList mWordTags @@ -406,10 +407,10 @@ public class WordNet { */ private void addWordSense(WordTagID wordTagId, int num) { num--; // convert to 0-indexed - assert num>=0 && (mWordTags.size()<=num || - mWordTags.get(num)==null || mWordTags.get(num)==wordTagId) - : "Unexpected repeat of word "+wordTagId+" on "+this - +": already "+mWordTags.get(num); + assert num >= 0 && (mWordTags.size() <= num || + mWordTags.get(num) == null || mWordTags.get(num) == wordTagId) + : "Unexpected repeat of word " + wordTagId + " on " + this + + ": already " + mWordTags.get(num); while (mWordTags.size() <= num) { mWordTags.add(null); @@ -419,7 +420,7 @@ public class WordNet { @Override public String toString() { - return mWordTags.get(0)+"#"+numberInFirstWordTagID; + return mWordTags.get(0) + "#" + numberInFirstWordTagID; } } @@ -475,7 +476,7 @@ public class WordNet { if (split >= 1) { int num; try { - num = Integer.parseInt(string.substring(split+1)); + num = Integer.parseInt(string.substring(split + 1)); } catch (NumberFormatException e) { return null; } @@ -599,35 +600,35 @@ public class WordNet { /** Checks the representation invariants */ private void checkrep() { for (EdgeType edgetype : EdgeType.values()) { - assertIt(loadedEdges.contains(edgetype), "Failed to load "+edgetype); + assertIt(loadedEdges.contains(edgetype), "Failed to load " + edgetype); } for (SynsetID synset : synsets.values()) { - assertIt(all.get(synset.index()) == synset, "Misplaced synset "+synset); + assertIt(all.get(synset.index()) == synset, "Misplaced synset " + synset); assertIt(synset.getWordTag(1).getSynset(synset.numberInFirstWordTagID) == synset, - "Wrong number in first WordTagID "+synset); - assertIt(!synset.wordtags.contains(null), synset+" contains null WordTagID"); + "Wrong number in first WordTagID " + synset); + assertIt(!synset.wordtags.contains(null), synset + " contains null WordTagID"); for (WordTagID wordtag : synset.wordtags) { - assertIt(wordtag.synsets.contains(synset), "Miswired Synset "+synset); + assertIt(wordtag.synsets.contains(synset), "Miswired Synset " + synset); } } for (WordTagID wordtag : wordtags.values()) { - assertIt(all.get(wordtag.index()) == wordtag, "Misplaced wordtag "+wordtag); - assertIt(!wordtag.synsets.contains(null), wordtag+" contains null SynsetID"); + assertIt(all.get(wordtag.index()) == wordtag, "Misplaced wordtag " + wordtag); + assertIt(!wordtag.synsets.contains(null), wordtag + " contains null SynsetID"); for (SynsetID synset : wordtag.synsets) { - assertIt(synset.wordtags.contains(wordtag), "Miswired WordTag "+wordtag); + assertIt(synset.wordtags.contains(wordtag), "Miswired WordTag " + wordtag); } - assertIt(wordtag.word.wordTags.containsValue(wordtag), "Miswired WordTag "+wordtag); + assertIt(wordtag.word.wordTags.containsValue(wordtag), "Miswired WordTag " + wordtag); } for (WordID word : words.values()) { - assertIt(all.get(word.index()) == word, "Misplaced word "+word); + assertIt(all.get(word.index()) == word, "Misplaced word " + word); for (WordTagID wordtag : word.wordTags.values()) { - assertIt(wordtag.word == word, "Miswired WordTag "+word); + assertIt(wordtag.word == word, "Miswired WordTag " + word); } } @@ -635,8 +636,8 @@ public class WordNet { for (EdgeType[] pair : transpose) { for (WordNetID id2 : id1.get(pair[0])) { assertIt(id2.get(pair[1]).contains(id1), - "Missing transpose "+Arrays.asList(transpose) - +" "+id1+" "+id2); + "Missing transpose " + Arrays.asList(transpose) + + " " + id1 + " " + id2); } } } @@ -645,7 +646,7 @@ public class WordNet { for (EdgeType type : reflexive) { for (WordNetID id2 : id1.get(type)) { assertIt(id2.get(type).contains(id1), - "Missing reflextive "+type+" "+id1+" "+id2); + "Missing reflextive " + type + " " + id1 + " " + id2); } } } @@ -672,11 +673,10 @@ public class WordNet { * currently supports only Prolog DB format. */ public static WordNet load(File path) { - System.err.println("WordNet.load: "+path); + System.err.println("WordNet.load: " + path); // see if already loaded - for (Iterator> it = instances.iterator(); - it.hasNext(); ) { + for (Iterator> it = instances.iterator(); it.hasNext();) { WordNet wordnet = it.next().get(); @@ -706,7 +706,7 @@ public class WordNet { WordNet wordnet = new WordNet(path); /** Global stashing point for unassigned word senses depending on wn POS tag */ - Map deferredPositionSynsets = new HashMap(); + Map deferredPositionSynsets = new HashMap(); // // read the synsets file @@ -716,14 +716,14 @@ public class WordNet { if (line.length() == 0) continue; // fields from the line - String[] fields = line.substring(2, line.length()-2).split(","); + String[] fields = line.substring(2, line.length() - 2).split(","); final int wordTagNumberInSynset = Integer.parseInt(fields[1]); final int synsetNumberInWordTag = Integer.parseInt( fields.length > 4 ? fields[4] : "0"); final int senseCount = Integer.parseInt( fields.length > 5 ? fields[5] : "0"); final String word = new String(fields[2].substring( - 1,fields[2].length()-1).replaceAll("\\s+", "_")) + 1, fields[2].length() - 1).replaceAll("\\s+", "_")) .toLowerCase(); final PartOfSpeech tag = PartOfSpeech.fromWordNetSSType( fields[3].charAt(0)); @@ -771,7 +771,7 @@ public class WordNet { } // add in deferredPositionSynsets - for (Map.Entry entry : deferredPositionSynsets.entrySet()) { + for (Map.Entry entry : deferredPositionSynsets.entrySet()) { boolean placed = false; for (int i = 0; i < entry.getKey().mSynsets.size(); i++) { if (entry.getKey().mSynsets.get(i) == null) { @@ -779,14 +779,14 @@ public class WordNet { placed = true; } } - if ( ! placed) { + if (!placed) { throw new AssertionError("Unable to place deferred synset"); } } // tell each SynsetID its position in its first WordTagID for (SynsetID synset : wordnet.synsets.values()) { - int position = synset.wordtags.get(0).mSynsets.indexOf(synset)+1; + int position = synset.wordtags.get(0).mSynsets.indexOf(synset) + 1; assert position >= 1 : "Unexpected: couldn't find the synset"; synset.numberInFirstWordTagID = position; } @@ -845,13 +845,13 @@ public class WordNet { */ private void loadSynsetRelation(File path, String relation, EdgeType type) { if (loadedEdges.contains(type)) { - throw new IllegalArgumentException("Unexpected error: trying to load "+type+" twice"); + throw new IllegalArgumentException("Unexpected error: trying to load " + type + " twice"); } loadedEdges.add(type); - for (String line : ObjectBank.getLineIterator(new File(path,"wn_"+relation+".pl"))) { + for (String line : ObjectBank.getLineIterator(new File(path, "wn_" + relation + ".pl"))) { if (line.length() == 0) continue; - String[] fields = line.substring(relation.length()+1, line.length()-2).split(","); + String[] fields = line.substring(relation.length() + 1, line.length() - 2).split(","); SynsetID id1 = getSynsetID(fields[0]); SynsetID id2 = getSynsetID(fields[1]); @@ -866,35 +866,21 @@ public class WordNet { */ private void loadWordRelation(File path, String relation, EdgeType type) { if (loadedEdges.contains(type)) { - throw new IllegalArgumentException("Unexpected error: trying to load "+type+" twice"); + throw new IllegalArgumentException("Unexpected error: trying to load " + type + " twice"); } loadedEdges.add(type); - for (String line : ObjectBank.getLineIterator(new File(path,"wn_"+relation+".pl"))) { + for (String line : ObjectBank.getLineIterator(new File(path, "wn_" + relation + ".pl"))) { if (line.length() == 0) continue; - String[] fields = line.substring(relation.length()+1, line.length()-2).split(","); + String[] fields = line.substring(relation.length() + 1, line.length() - 2).split(","); final SynsetID sid1 = getSynsetID(fields[0]); final SynsetID sid2 = getSynsetID(fields[2]); -// WordNetID id1 = sid1, id2 = sid2; -// -// final int num1 = Integer.parseInt(fields[1]); -// final int num2 = Integer.parseInt(fields[3]); -// -// if ((num1 == 0) ^ (num2 == 0)) { -// throw new RuntimeException("Unexpected error in "+relation+": one zero wordnum (should be none or both)"); -// } else if (num1 != 0 && num2 != 0) { -// id1 = sid1.getWordTag(num1); -// id2 = sid2.getWordTag(num2); -// } - if (sid1 == sid2) { - System.err.println("WordNet.loadWordRelation("+relation+"): skipping self-loop on "+sid1); + System.err.println("WordNet.loadWordRelation(" + relation + "): skipping self-loop on " + sid1); } else { - // TODO: choose between word id and synset id sid1.add(type, sid2); - // id1.add(type, id2); } } } @@ -910,19 +896,19 @@ public class WordNet { loadedEdges.contains(EdgeType.TERM_HAS_USAGE) || loadedEdges.contains(EdgeType.TERM_IN_REGION)) { - throw new IllegalArgumentException("Unexpected error while loading "+relation); + throw new IllegalArgumentException("Unexpected error while loading " + relation); } loadedEdges.add(EdgeType.TERM_HAS_TOPIC); loadedEdges.add(EdgeType.TERM_HAS_USAGE); loadedEdges.add(EdgeType.TERM_IN_REGION); - for (String line : ObjectBank.getLineIterator(new File(path,"wn_"+relation+".pl"))) { + for (String line : ObjectBank.getLineIterator(new File(path, "wn_" + relation + ".pl"))) { if (line.length() == 0) continue; - String[] fields = line.substring(relation.length()+1, line.length()-2).split(","); + String[] fields = line.substring(relation.length() + 1, line.length() - 2).split(","); if (fields.length != 5 || fields[4].length() != 1) { - throw new IllegalArgumentException("Badly formed file for "+relation); + throw new IllegalArgumentException("Badly formed file for " + relation); } SynsetID sid1 = getSynsetID(fields[0]); @@ -935,7 +921,7 @@ public class WordNet { WordNetID id1 = num1 == 0 ? sid1 : sid1.getWordTag(num1); WordNetID id2 = num2 == 0 ? sid2 : sid2.getWordTag(num2); - switch(fields[4].charAt(0)) { + switch (fields[4].charAt(0)) { case 't': id1.add(EdgeType.TERM_HAS_TOPIC, id2); break; @@ -949,7 +935,7 @@ public class WordNet { break; default: - throw new IllegalArgumentException("Unexpected relation type "+fields[4]); + throw new IllegalArgumentException("Unexpected relation type " + fields[4]); } } } @@ -957,9 +943,9 @@ public class WordNet { /** Adds a new edge type t2 as the transpose of t1 */ private void addTranspose(EdgeType t1, EdgeType t2) { if (!loadedEdges.contains(t1)) { - throw new IllegalArgumentException("Cannot transpose: doesn't contain "+t1); + throw new IllegalArgumentException("Cannot transpose: doesn't contain " + t1); } else if (loadedEdges.contains(t2)) { - throw new IllegalArgumentException("Cannot transpose: already contains "+t2); + throw new IllegalArgumentException("Cannot transpose: already contains " + t2); } loadedEdges.add(t2); @@ -979,13 +965,13 @@ public class WordNet { WordNet wordnet = load(new File(args[0])); SynsetID id1 = wordnet.getSynsetID("run#v#1"); - + for (EdgeType edgetype : EdgeType.values()) { for (WordNetID id2 : id1.get(edgetype)) { System.out.println(edgetype + " " + id2); } } - + System.out.println(wordnet.getAllWordNetIDs().size()); for (SynsetID synset : wordnet.synsets.values()) { diff --git a/src/edu/stanford/nlp/sempre/paraphrase/Aligner.java b/src/edu/stanford/nlp/sempre/paraphrase/Aligner.java deleted file mode 100644 index 0950fca..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/Aligner.java +++ /dev/null @@ -1,383 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import edu.stanford.nlp.sempre.BooleanValue; -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.LanguageInfo.WordInfo; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.LanguageInfo.LanguageUtils; -import edu.stanford.nlp.sempre.fbalignment.utils.WnExpander; -import edu.stanford.nlp.sempre.paraphrase.paralex.PhraseTable; -import edu.stanford.nlp.sempre.paraphrase.rules.RuleApplication; -import edu.stanford.nlp.sempre.paraphrase.rules.RuleApplier; -import edu.stanford.nlp.sempre.paraphrase.rules.Rulebase; -import fig.basic.IntPair; -import fig.basic.LispTree; -import fig.basic.LogInfo; -import fig.basic.Option; - -/** - * Exhaustive alignment between two paraphrases - */ -public class Aligner { - - public static class Options { - @Option(gloss="Path to file with derivations") public String derivationPath="lib/derivation.txt"; - @Option(gloss="Maximum distortion") public double distortionParam=1; - @Option(gloss="Whether to use syntax rules") public boolean useSyntax=false; - @Option(gloss="Whether to use wordnet synsets") public boolean useWordnet=false; - @Option public int verbose=0; - } - public static Options opts = new Options(); - private static Aligner aligner; - - private final PhraseTable phraseTable; - private final Map> derivations; - private Rulebase ruleBase; - private WnExpander wnExpander; - - - private Aligner() throws IOException { - phraseTable = PhraseTable.getSingleton(); - derivations = ParaphraseUtils.loadDerivations(opts.derivationPath); - wnExpander = new WnExpander(); - if(opts.useSyntax) - ruleBase = new Rulebase(); - } - - public static Aligner getSingleton() throws IOException { - if(aligner==null) - aligner = new Aligner(); - return aligner; - } - - public Alignment align(ParaphraseExample example, Params params) { - example.ensureAnnotated(); - Alignment alignment = new Alignment(example); - alignment.buildAlignment(example, params); - example.setAlignment(alignment); - return alignment; - } - - public class Alignment { - - private List substitutions = new ArrayList(); - private List deletions = new ArrayList(); - FeatureVector featureVector = new FeatureVector(); - double score = Double.NaN; - private final String source; - private final String target; - - public Alignment(ParaphraseExample example) { - source = example.source; - target = example.target; - } - - public void buildAlignment(ParaphraseExample example, Params params) { - computeIdentityAlignments(example); - computePhraseTableAlignments(example); - computeSubstitutionsAlignment(example); - computeDerivationsAlignment(example); - //this needs to be done last - markDeletions(example); - if(opts.useSyntax) - computeSyntacticAlignment(example); - if(opts.verbose>=1) { - printFeaturesAndWeights(params); - } - score = featureVector.dotProduct(params); - } - - private void printFeaturesAndWeights(Params params) { - for(String key: featureVector.toMap().keySet()) { - double value = featureVector.toMap().get(key); - LogInfo.logs("Printing features: feautre=%s, value=%s, weight=%s,product=\t%s",key,value, - params.getWeight(key),value * params.getWeight(key)); - } - } - - private void computeSyntacticAlignment(ParaphraseExample example) { - for(RuleApplier rule: ruleBase.getRules()) { - for(RuleApplication application: rule.apply(example.sourceInfo, example.targetInfo)) { - List wordInfos = new ArrayList(); - for(int i = 0; i < application.consequent.numTokens(); ++i) - wordInfos.add(application.consequent.getWordInfo(i)); - if(example.targetInfo.matchLemmas(wordInfos)) { - featureVector.add("SyntAlign",rule.toString()); - } - } - } - } - - private void computeDerivationsAlignment(ParaphraseExample example) { - for(int i = 0 ; i < example.sourceInfo.numTokens(); ++i) { - for(int j = 0; j < example.targetInfo.numTokens(); ++j) { - String sourceLemma = example.sourceInfo.lemmaTokens.get(i); - String targetLemma = example.targetInfo.lemmaTokens.get(j); - if(!sourceLemma.equals(targetLemma)) { - if(derivations.containsKey(sourceLemma) && - derivations.get(sourceLemma).contains(targetLemma)) { - AlignmentInterval sourceInterval = - new AlignmentInterval(sourceLemma, new Interval(i, i+1)); - AlignmentInterval targetInterval = - new AlignmentInterval(targetLemma, new Interval(j, j+1)); - substitutions.add(new AlignmentIntervalPair(sourceInterval, targetInterval)); - featureVector.add("Subst", "Deriv"); - featureVector.add("Subst", "l="+sourceLemma+",r="+targetLemma); - } - } - } - } - } - - private void markDeletions(ParaphraseExample example) { - markDeletion(example.sourceInfo,true); - markDeletion(example.targetInfo,false); - } - - private void markDeletion(LanguageInfo lInfo, boolean isSource) { - - for(int i = 0; i < lInfo.numTokens(); ++i) { - boolean aligned=false; - for(AlignmentIntervalPair alignmentIntervalPair: substitutions) { - AlignmentInterval alignmentInterval = isSource ? alignmentIntervalPair.sourceInterval - : alignmentIntervalPair.targetInterval; - if(alignmentInterval.interval.contains(i)) { - aligned=true; - break; - } - } - if(!aligned) { - AlignmentInterval deletedToken = new AlignmentInterval(lInfo.lemmaTokens.get(i), new Interval(i, i+1)); - deletions.add(deletedToken); - if(ParaphraseFeatureMatcher.containsDomain("Del")) { - featureVector.add("Del", "lemma="+deletedToken.phrase); - featureVector.add("Del", "pos="+LanguageUtils.getCanonicalPos(lInfo.posTags.get(i))); - } - } - } - } - - private void computeIdentityAlignments(ParaphraseExample example) { - for(int i = 0 ; i < example.sourceInfo.numTokens(); ++i) { - for(int j = 0; j < example.targetInfo.numTokens(); ++j) { - if(example.sourceInfo.lemmaTokens.get(i).equals(example.targetInfo.lemmaTokens.get(j))) { - - AlignmentInterval sourceInterval = - new AlignmentInterval(example.sourceInfo.lemmaTokens.get(i), new Interval(i, i+1)); - AlignmentInterval targetInterval = - new AlignmentInterval(example.targetInfo.lemmaTokens.get(j), new Interval(j, j+1)); - substitutions.add(new AlignmentIntervalPair(sourceInterval, targetInterval)); - featureVector.add("Subst", "Identity", 1); - } - } - } - } - - /** - * We consider all tokens and phrases with two nouns and allow substitutions if canonical pos matches. - * @param example - */ - private void computeSubstitutionsAlignment(ParaphraseExample example) { - - List sourceIntervals = getLanguageInfoCandidates(example.sourceInfo); - List targetIntervals = getLanguageInfoCandidates(example.targetInfo); - for(Interval sourceInterval: sourceIntervals) { - for(Interval targetInterval: targetIntervals) { - String sourcePhrase = example.sourceInfo.lemmaPhrase(sourceInterval.start, sourceInterval.end); - String targetPhrase = example.targetInfo.lemmaPhrase(targetInterval.start, targetInterval.end); - if(!sourcePhrase.equals(targetPhrase) && - ParaphraseUtils.posCompatible(example.sourceInfo.posTags.get(sourceInterval.start), - example.targetInfo.posTags.get(targetInterval.start))) { - - AlignmentInterval sourceAlignmentInterval = new AlignmentInterval(sourcePhrase, sourceInterval); - AlignmentInterval targetAlignmentInterval = new AlignmentInterval(targetPhrase, targetInterval); - if(validDistortion(sourceAlignmentInterval, example.sourceInfo, - targetAlignmentInterval,example.targetInfo)) { - substitutions.add(new AlignmentIntervalPair(sourceAlignmentInterval, targetAlignmentInterval)); - //features - String lPos = example.sourceInfo.canonicalPosSeq(sourceInterval.start, sourceInterval.end); - String rPos = example.targetInfo.posSeq(targetInterval.start, targetInterval.end); - if(!lPos.equals(rPos)) - featureVector.add("Subst", "l_pos="+lPos+",r_pos="+rPos); - else - featureVector.add("Subst", "pos_identity"); - featureVector.add("Subst", "l="+sourcePhrase+",r="+targetPhrase); - if(opts.useWordnet) { - if(wnExpander.getSynonyms(sourcePhrase).contains(targetPhrase)) - featureVector.add("Subst", "synonym"); - } - } - } - } - } - } - - private boolean validDistortion(AlignmentInterval sourceAlignmentInterval, - LanguageInfo sourceInfo, AlignmentInterval targetAlignmentInterval, LanguageInfo targetInfo) { - - double sourcePosition = sourceAlignmentInterval.interval.middle() / sourceInfo.numTokens(); - double targetPosition = targetAlignmentInterval.interval.middle() / targetInfo.numTokens(); - int min = Math.min(sourceInfo.numTokens(), targetInfo.numTokens()); - return (Math.abs(sourcePosition-targetPosition) getLanguageInfoCandidates(LanguageInfo lInfo) { - List res = new ArrayList(); - for(int i = 0; i < lInfo.numTokens(); ++i) { - res.add(new Interval(i, i+1)); - if(i < lInfo.numTokens()-1 && - LanguageUtils.isNN(lInfo.posTags.get(i)) && - LanguageUtils.isNN(lInfo.posTags.get(i+1))) { - res.add(new Interval(i,i+2)); - } - } - return res; - } - - private void computePhraseTableAlignments(ParaphraseExample example) { - //go over all lhs spans - for(int i = 0; i < example.sourceInfo.numTokens(); ++i) { - for(int j = i+1; j <= example.sourceInfo.numTokens(); ++j) { - String lhsPhrase = example.sourceInfo.lemmaPhrase(i, j); - if(phraseTable.containsKey(lhsPhrase)) { - Map rhsCandidates = phraseTable.get(lhsPhrase); - //get rhs - Map rhsLemmaSpans = example.targetInfo.getLemmaSpans(); - for(String rhsLemmaSpan: rhsLemmaSpans.keySet()) { - if(rhsCandidates.containsKey(rhsLemmaSpan)) { - int targetStart = rhsLemmaSpans.get(rhsLemmaSpan).first; - int targetEnd = rhsLemmaSpans.get(rhsLemmaSpan).second; - AlignmentStats aStats = rhsCandidates.get(rhsLemmaSpan); - - //now we have the source phrase the target phrase and the intervals - AlignmentInterval sourceInterval = new AlignmentInterval(lhsPhrase, new Interval(i, j)); - AlignmentInterval targetInterval = new AlignmentInterval(rhsLemmaSpan, new Interval(targetStart, targetEnd)); - if(validDistortion(sourceInterval, example.sourceInfo, - targetInterval,example.targetInfo)) { - substitutions.add(new AlignmentIntervalPair(sourceInterval, targetInterval)); - //features - featureVector.add("Subst", "l="+lhsPhrase+",r="+rhsLemmaSpan); - if(ParaphraseFeatureMatcher.containsDomain("Pt")) { - featureVector.add("Pt","cooccurCount="+binCount(aStats.cooccurrenceCount)); - featureVector.add("Pt","phraseCount="+binCount(aStats.phrase1Count)); - featureVector.add("Pt","phraseCount="+binCount(aStats.phrase2Count)); - } - String lPos = example.sourceInfo.canonicalPosSeq(i, j); - String rPos = example.targetInfo.canonicalPosSeq(targetStart, targetEnd); - if(lPos.equals(rPos)) - featureVector.add("Subst","pos_identity"); - else - featureVector.add("Subst", "l_pos="+lPos+",r_pos="+rPos); - } - } - } - } - } - } - } - - private String binCount(double count) { - if(count<=50) - return "<=50"; - if(count<=100) - return "<=100"; - if(count<=500) - return "<=500"; - if(count<=1000) - return "<=1000"; - if(count<=5000) - return "<=5000"; - if(count<=10000) - return "<=10000"; - if(count<=50000) - return "<=50000"; - if(count<=100000) - return "<=100000"; - if(count<=1000000) - return "<=1000000"; - return ">1000000"; - } - - public void clear() { - featureVector.clear(); - substitutions.clear(); - deletions.clear(); - } - - public LispTree toLispTree() { - LispTree tree = LispTree.proto.newList(); - tree.addChild("alignment"); - tree.addChild(LispTree.proto.newList("source", ""+source)); - tree.addChild(LispTree.proto.newList("target", ""+target)); - tree.addChild(LispTree.proto.newList("alignment_score", ""+score)); - return tree; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - for(String key: featureVector.toMap().keySet()) { - sb.append(key+"\t"+featureVector.toMap().get(key)+"\n"); - } - return sb.toString(); - } - } - - public class AlignmentInterval { - public final String phrase; - public final Interval interval; - public AlignmentInterval(String phrase, Interval interval) { - super(); - this.phrase = phrase; - this.interval = interval; - } - public String toString() { - return phrase+","+interval; - } - } - public class AlignmentIntervalPair { - public final AlignmentInterval sourceInterval; - public final AlignmentInterval targetInterval; - public AlignmentIntervalPair(AlignmentInterval sourceInterval, - AlignmentInterval targetInterval) { - super(); - this.sourceInterval = sourceInterval; - this.targetInterval = targetInterval; - } - public String toString() { - return sourceInterval+"\t"+targetInterval; - } - } - - public static class AlignmentStats { - public final double cooccurrenceCount; - public final double phrase1Count; - public final double phrase2Count; - - public AlignmentStats(double cooccurrence, double phrase1Freq, double phrase2Freq) { - this.cooccurrenceCount=cooccurrence; - this.phrase1Count=phrase1Freq; - this.phrase2Count=phrase2Freq; - } - - } - - public static void main(String[] args) throws IOException { - - ParaphraseExample paraExample =new ParaphraseExample("what type of music did richard wagner play ?", - "what is the musical genres of richard wagner ?",new BooleanValue(true)); - Aligner aligner = new Aligner(); - Params params = new Params(); - params.read("/Users/jonathanberant/Research/temp/params"); - Alignment alignment = aligner.align(paraExample, params); - alignment.printFeaturesAndWeights(params); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/Context.java b/src/edu/stanford/nlp/sempre/paraphrase/Context.java deleted file mode 100644 index d50cd0c..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/Context.java +++ /dev/null @@ -1,98 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Joiner; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * The context of a phrase represented as |lhs| and |rhs| - */ -public class Context { - - @JsonProperty public final String lhs; - @JsonProperty public final String rhs; - private Set lhsTokens; - private Set rhsTokens; - - public Context(String lhs,String rhs) { - this.lhs = lhs; - this.rhs = rhs; - String[] lhsArray = lhs.split("\\s+"); - String[] rhsArray = rhs.split("\\s+"); - lhsTokens = new HashSet(Arrays.asList(lhsArray)); - rhsTokens = new HashSet(Arrays.asList(rhsArray)); - } - - public Context(List tokens, Interval interval) { - lhs = Joiner.on(' ').join(tokens.subList(0, interval.start)); - rhs = Joiner.on(' ').join(tokens.subList(interval.end, tokens.size())); - String[] lhsArray = lhs.split("\\s+"); - String[] rhsArray = rhs.split("\\s+"); - lhsTokens = new HashSet(Arrays.asList(lhsArray)); - rhsTokens = new HashSet(Arrays.asList(rhsArray)); - } - - @JsonCreator - public Context(@JsonProperty("context") String context) { - String[] tokens = context.split("__"); - this.lhs = tokens[0].substring(1); - this.rhs = tokens[1].substring(0,tokens[1].length()-1); - String[] lhsArray = lhs.split("\\s+"); - String[] rhsArray = rhs.split("\\s+"); - lhsTokens = new HashSet(Arrays.asList(lhsArray)); - rhsTokens = new HashSet(Arrays.asList(rhsArray)); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((lhs == null) ? 0 : lhs.hashCode()); - result = prime * result + ((rhs == null) ? 0 : rhs.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - Context other = (Context) obj; - if (lhs == null) { - if (other.lhs != null) - return false; - } else if (!lhs.equals(other.lhs)) - return false; - if (rhs == null) { - if (other.rhs != null) - return false; - } else if (!rhs.equals(other.rhs)) - return false; - return true; - } - - @Override - public String toString() { - return "["+lhs+"__"+rhs+"]"; - } - - public String toUtteranceString() { - return lhs + " X " + rhs; - } - - public Set getLhsTokens() { - return lhsTokens; - } - - public Set getRhsTokens() { - return rhsTokens; - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ContextModel.java b/src/edu/stanford/nlp/sempre/paraphrase/ContextModel.java deleted file mode 100644 index 0b2f8a3..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ContextModel.java +++ /dev/null @@ -1,314 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import edu.stanford.nlp.sempre.*; -import edu.stanford.nlp.sempre.fbalignment.utils.DoubleContainer; -import edu.stanford.nlp.sempre.fbalignment.utils.MathUtils; -import fig.basic.*; - -import java.io.File; -import java.util.*; - -/** - * Maps a context to a distribution over logical forms - * @author jonathanberant - * - */ -public class ContextModel { - - public static class Options { - @Option public int verbose = 3; - @Option public String packageName = "edu.stanford.nlp.sempre.paraphrase"; - @Option public String contextModelName; - @Option public double minCosine=0.75; - } - public static Options opts = new Options(); - - @JsonProperty private Map> contextToBinaryCounter; - private boolean normalized=false; - private FbFormulasInfo fbFormulasInfo; - private ContextSimilarityModel transModel; - - public ContextModel() { - contextToBinaryCounter = new HashMap<>(); - fbFormulasInfo = FbFormulasInfo.getSingleton(); - try { - transModel = (ContextSimilarityModel) Class.forName(opts.packageName+"."+opts.contextModelName).newInstance(); - } - catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { - throw new RuntimeException(e); - } - } - - @JsonCreator - public ContextModel(@JsonProperty("contextToBinaryCounter") Map> contextToBinaryCounter) { - this.contextToBinaryCounter = contextToBinaryCounter; - fbFormulasInfo = FbFormulasInfo.getSingleton(); - try { - transModel = (ContextSimilarityModel) Class.forName(opts.packageName+"."+opts.contextModelName).newInstance(); - } - catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - public void inc(Context c, Formula f, double value) { - if(normalized) - throw new RuntimeException("Can not increment a normalized context mapper"); - - MapUtils.putIfAbsent(contextToBinaryCounter, c, new HashMap()); - Map formulaMap = contextToBinaryCounter.get(c); //guaranteed to exist - - MapUtils.putIfAbsent(formulaMap, f, new DoubleContainer(0.0)); - DoubleContainer currValue = formulaMap.get(f); - currValue.inc(value); - - } - - public void normalize() { - normalized = true; - LogInfo.logs("ContextModel.normalize: number of contexts=%s",contextToBinaryCounter.size()); - - for(Context context: contextToBinaryCounter.keySet()) { - Map formulaCounter = contextToBinaryCounter.get(context); - MathUtils.normalizeDoubleMap(formulaCounter); - } - } - - public boolean containsKey(Context c) { - return contextToBinaryCounter.containsKey(c); - } - - public void map(Example ex) { - - Map>> formulaToContextAndInterval = new HashMap<>(); - //go over all derivations and check which one contains a correct formula - for(Derivation deriv: ex.getPredDerivations()) { - if(ex.targetValue.getCompatibility(deriv.getValue())>0.999) { - Formula f = extractFormula(deriv.formula); - processCurrDerivContext(formulaToContextAndInterval, ParaphraseUtils.extractContextIntervalPair(deriv, ex),f); - } - } - - for(Formula f: formulaToContextAndInterval.keySet()) { - for(Pair pair: formulaToContextAndInterval.get(f)) { - LogInfo.logs("ContextModel: context=%s, formula=%s",pair.getFirst(),f); - inc(pair.getFirst(), f, 1.0); - } - } - } - - - private void processCurrDerivContext(Map>> formulaToContextAndInterval, - Pair contextIntervalPair, Formula f) { - - Pair newPair = Pair.newPair(contextIntervalPair.getFirst(),contextIntervalPair.getSecond()); - List> existingformulaContexts = formulaToContextAndInterval.get(f); - if(existingformulaContexts==null) { - existingformulaContexts = new LinkedList<>(); - formulaToContextAndInterval.put(f, existingformulaContexts); - } - for(Pair existingFormulaContext: existingformulaContexts) { - if(existingFormulaContext.getSecond().superset(newPair.getSecond())) { - if(opts.verbose>=3) { - LogInfo.logs("ContextModel: skipping context, context=%s, interval=%s, otherContext=%s, otherInterval=%s",newPair.getFirst(), - newPair.getSecond(),existingFormulaContext.getFirst(),existingFormulaContext.getSecond()); - } - return; - } - } - existingformulaContexts.add(newPair); - } - - private Formula extractFormula(Formula formula) { - if(!(formula instanceof JoinFormula)) - throw new RuntimeException("ContextModel: logical form is not a join formula: " + formula); - JoinFormula join = (JoinFormula) formula; - return Formulas.fromLispTree(join.relation.toLispTree()); - } - - /** - * Given an example and the context, compare the context to all contexts we have and find the best formula for it - * then apply this formula on the entity - * @param ex - * @param entityInstancesScores - */ - public List computePredictions(Example ex, TDoubleMap entityInstancesScores) { - - LogInfo.begin_track("ContextModel.computePredictions()"); - TDoubleMap formulaScores = new TDoubleMap<>(); - //for every possible entity - for(EntityInstance entityInstance: entityInstancesScores.keySet()) { - - //for every context - Map contextDist = transModel.getContextDist(contextToBinaryCounter.keySet(), entityInstance.context); - for(Context currContext: contextDist.keySet()) { - - Map contextFormulas = MapUtils.get(contextToBinaryCounter, currContext, new HashMap()); - double contextProb = contextDist.get(currContext); - LogInfo.logs("ContextModel: context=%s, context probability=%s, number of formulas=%s",currContext, contextProb,contextFormulas.size()); - - //for each formula compute its probability - calcFormulaProbsGivenContextAndEntity(entityInstance,entityInstancesScores.get(entityInstance, 0.0), contextProb, contextFormulas,formulaScores); - } - } - //after computing everything we need to sort and return - List res = new ArrayList(); - for(Formula f: formulaScores.keySet()) { - res.add(new Prediction(f, formulaScores.get(f, 0.0))); - } - Collections.sort(res); - if(opts.verbose>=3) { - LogInfo.logs("ContextModel: number of type compatible formulas=%s",res.size()); - for(Prediction p: res) - LogInfo.logs("ContextModel: prediction=%s, score=%s",p.formula, p.score); - } - LogInfo.end_track(); - return res; - } - - /** - * Formula probability is computed by using the model, zeroing relations that are not - * compatible with the entity and re-normalizing - */ - private void calcFormulaProbsGivenContextAndEntity( - EntityInstance entityInstance, double entityProb, double contextProb, Map contextRelations, TDoubleMap formulaScores) { - - Formula[] relations = new Formula[contextRelations.size()]; - double[] scores = new double[contextRelations.size()]; - - int i = 0; - //zero non-compatible scores - for(Formula currRelation: contextRelations.keySet()) { - - //see if semantic types match - FbFormulasInfo.BinaryFormulaInfo bInfo = fbFormulasInfo.getBinaryInfo(currRelation); - if(bInfo==null) - throw new RuntimeException("No info for relation: " + currRelation); - - SemType entityType = entityInstance.semType; - SemType binaryType = bInfo.getSemType(); - SemType type = binaryType.apply(entityType); - boolean isValid = type.isValid(); - if(isValid) - LogInfo.logs("ContextModel: type compatible formula=%s", currRelation); - else - LogInfo.logs("ContextModel: type incompatible formula=%s, binrayType=%s", currRelation,binaryType); - - relations[i]=currRelation; - scores[i++] = isValid ? contextRelations.get(currRelation).value() : 0.0; - } - //re-normalize - NumUtils.normalize(scores); - //populate - for(int j = 0; j < relations.length; ++j) { - if(scores[j]>0.0) { - Formula jFormula = new JoinFormula(relations[j], entityInstance.formula); //create the full join formula from the context and the entity - if(opts.verbose>=3) - //here we integrate all 3 scores - entity score: p(c_x,s_x,e|x), context score: p(c|c_x), binary score: p(b|c,e) - formulaScores.incr(jFormula, entityProb * contextProb * scores[j]); - } - } - } - - public void log() { - LogInfo.begin_track("Logging contexts"); - for(Context c: contextToBinaryCounter.keySet()) { - for(Formula f: contextToBinaryCounter.get(c).keySet()) { - LogInfo.log("ContextModel: context to formula:\t" + c + "\t"+ f + "\t" + contextToBinaryCounter.get(c).get(f).value()); - } - } - LogInfo.end_track(); - } - - /** - * Generate a training set from the map by treating contexts c1 and c2 as paraphrases - * if the formulas they are mapped with highest score is equal - */ - public void generateTrainingSet(String outFile) { - - Map> formulaToContextMap = new HashMap<>(); - - LogInfo.log("Convert map context-->formulas to map formula-->contexts"); - LogInfo.logs("number of contexts: %s",contextToBinaryCounter.size()); - for(Context c: contextToBinaryCounter.keySet()) { - double topScore = 0.0; - Set bestFormulas = new HashSet<>(); - //collect the best formulas - for(Formula f: contextToBinaryCounter.get(c).keySet()) { - double currScore = contextToBinaryCounter.get(c).get(f).value(); - if(currScore-topScore>1e-10) { //if current formula is better than best so far - bestFormulas.clear(); - bestFormulas.add(f); - topScore = currScore; - } - else if(Math.abs(currScore)-topScore < 1e-10) { // if current formulas is as good as best so far - bestFormulas.add(f); - } - } - //add the best formulas - for(Formula bestFormula: bestFormulas) { - MapUtils.addToList(formulaToContextMap, bestFormula, c); - } - } - LogInfo.logs("Number of formulas: %s",formulaToContextMap.size()); - - //generate positive training examples - generateExamples(formulaToContextMap, outFile); - } - - private void generateExamples(Map> formulaToContextMap, String outFile) { - List positives = new LinkedList(); - List negatives = new LinkedList(); - - double posAvgCosine=0d, negAvgCosine=0d; - int numOfNeg=0, numOfPos = 0; - List seenContexts = new LinkedList<>(); - int numOfFormulas=0; - for(Formula formula: formulaToContextMap.keySet()) { - List contexts = formulaToContextMap.get(formula); - //generate positives - for(int i = 0; i < contexts.size()-1; ++i) { - for(int j = i+1; j < contexts.size(); ++j) { - String utter1 = contexts.get(i).toUtteranceString(); - String utter2 = contexts.get(j).toUtteranceString(); - double cosine = MathUtils.tokensCosine(Arrays.asList(utter1.split("\\s+")),Arrays.asList(utter2.split("\\s+"))); - posAvgCosine += cosine; - if(cosine>opts.minCosine) - positives.add(new ParaphraseExample(utter1,utter2,formula,new BooleanValue(true))); - } - } - //generatve negatives - for(Context seenContext: seenContexts) { - for(Context currContext: contexts) { - String utter1 = seenContext.toUtteranceString(); - String utter2 = currContext.toUtteranceString(); - double cosine = MathUtils.tokensCosine(Arrays.asList(utter1.split("\\s+")),Arrays.asList(utter2.split("\\s+"))); - negAvgCosine+=cosine; - numOfNeg++; - if(cosine>opts.minCosine && cosine<1) - negatives.add(new ParaphraseExample(utter1,utter2,formula,new BooleanValue(false))); - } - } - seenContexts.addAll(contexts); - if(++numOfFormulas % 10 == 0) - LogInfo.logs("Number of formulas=%s",numOfFormulas); - } - - //assuming more negatives - Sample same number of negatives as positives - ListUtils.randomPermute(negatives, new Random(1)); - negatives = negatives.subList(0, positives.size()); - - //write with Json - LogInfo.logs("ContextMolde.generateTrainingSet: avgPosCosine=%s, avgNegCosine=%s",(posAvgCosine/numOfPos),(negAvgCosine/numOfNeg)); - LogInfo.logs("ContextModel.generateTrainingSet: number of positives: %s",positives.size()); - LogInfo.logs("ContextModel.generateTrainingSet: number of negatives: %s",negatives.size()); - - List dataset = new LinkedList<>(positives); - dataset.addAll(negatives); - ListUtils.randomPermute(dataset, new Random(1)); - LogInfo.logs("ContextModel.generateTrainingSet: number of examples: %s",dataset.size()); - Json.writeValueHard(new File(outFile), dataset); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ContextSimilarityModel.java b/src/edu/stanford/nlp/sempre/paraphrase/ContextSimilarityModel.java deleted file mode 100644 index 45de5bf..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ContextSimilarityModel.java +++ /dev/null @@ -1,149 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.fbalignment.utils.CollectionUtils; -import edu.stanford.nlp.sempre.fbalignment.utils.DoubleContainer; -import edu.stanford.nlp.sempre.fbalignment.utils.MathUtils; -import edu.stanford.nlp.sempre.paraphrase.paralex.ParalexRules; -import fig.basic.LogInfo; -import fig.basic.Option; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * Given a context from a question c_x (x is the question) provides a - * distribution over contexts p(c|c_x) - * @author jonathanberant - * - */ -public abstract class ContextSimilarityModel { - - public static class Options { - @Option public String bowMode="average_jaccard"; - } - public static Options opts = new Options(); - - public abstract Map getContextDist(Set candidates, Context questionContext); -} - -/** - * All probability mass is on a single context that has highest similarity according to some similarity metric - * @author jonathanberant - * - */ -class BowSimilarityModel extends ContextSimilarityModel { - - private static BowSimilarityComputer simComputer = getInstance(); - - private static BowSimilarityComputer getInstance() { - if(opts.bowMode.equals("average_jaccard")) - return new AverageJaccardSimilarityComputer(); - else if(opts.bowMode.equals("jaccard")) - return new JaccardSimilarityComputer(); - throw new RuntimeException("bow similarity computer mode illegal" + opts.bowMode); - } - - public Map getContextDist(Set candidates, Context questionContext) { - if(candidates.size()==0) - throw new RuntimeException("There must be at least one context candidate"); - - Context mostSimilartContext = null; - double similarity = 0.0; - for(Context candidate: candidates) { - - double currSimilarity = simComputer.similarity(questionContext,candidate); - if(mostSimilartContext==null || currSimilarity > similarity) { - mostSimilartContext = candidate; - similarity = currSimilarity; - } - } - LogInfo.logs("ContextSimilarityModel.findBestContext: bestContext=%s, similarity=%s",mostSimilartContext,similarity); - Map res = new HashMap<>(); - res.put(mostSimilartContext, 1.0); - return res; - } -} - -/** - * All probability mass is on a single context that is equal to the target question if one exists - * @author jonathanberant - * - */ -class IdentityModel extends ContextSimilarityModel { - - public Map getContextDist(Set candidates, Context questionContext) { - if(candidates.size()==0) - throw new RuntimeException("There must be at least one context candidate"); - - Map res = new HashMap<>(); - - for(Context candidate: candidates) { - if(candidate.equals(questionContext)) { - res.put(candidate, 1.0); - break; - } - } - LogInfo.logs("ContextIdentityModel.getContextDist: context dist=%s",res); - return res; - } -} - -/** - * Given a two contexts returns a match if a transformation rule maps one context to the other - * @author jonathanberant - * - */ -class WholeContextTransformationModel extends ContextSimilarityModel { - - private ParalexRules rules = new ParalexRules(); - - @Override - public Map getContextDist(Set candidates, - Context questionContext) { - Map res = new HashMap(); - - for(Context candidate: candidates) { - if(candidate.equals(questionContext)) { - LogInfo.logs("RuleTransformationModel: Exact match",questionContext,candidate); - res.clear(); - res.put(candidate, new DoubleContainer(1.0)); - break; - } - double matchCount = rules.match(questionContext,candidate).value(); - if(matchCount>0.0) { - LogInfo.logs("RuleTransformationModel: question=%s, match=%s, score=%s",questionContext,candidate,matchCount); - res.put(candidate, new DoubleContainer(matchCount)); - } - } - MathUtils.normalizeDoubleMap(res); - return CollectionUtils.doubleContainerToDoubleMap(res); - } -} - -abstract class BowSimilarityComputer { - public abstract double similarity(Context c1, Context c2); -} - -class JaccardSimilarityComputer extends BowSimilarityComputer { - @Override - public double similarity(Context c1, Context c2) { - Set tokens1 = new HashSet<>(); - tokens1.addAll(c1.getLhsTokens()); - tokens1.addAll(c1.getRhsTokens()); - Set tokens2 = new HashSet<>(); - tokens2.addAll(c2.getLhsTokens()); - tokens2.addAll(c2.getRhsTokens()); - return MathUtils.jaccard(tokens1, tokens2); - } -} - -class AverageJaccardSimilarityComputer extends BowSimilarityComputer { - @Override - public double similarity(Context c1, Context c2) { - double leftJaccard = MathUtils.jaccard(c1.getLhsTokens(), c2.getLhsTokens()); - double rightJaccard = MathUtils.jaccard(c1.getRhsTokens(), c2.getRhsTokens()); - return (leftJaccard+rightJaccard) / 2; - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/DatasetUtils.java b/src/edu/stanford/nlp/sempre/paraphrase/DatasetUtils.java deleted file mode 100644 index c1c9475..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/DatasetUtils.java +++ /dev/null @@ -1,29 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import fig.basic.Pair; -import fig.prob.SampleUtils; - -public class DatasetUtils { - - public static Pair,List> splitTrainFromDev(List originalExamples,double trainFrac, double devFrac, Random rand) { - int split1 = (int) (trainFrac * originalExamples.size()); - int split2 = (int) ((1 - devFrac) * originalExamples.size()); - int[] perm = SampleUtils.samplePermutation(rand, originalExamples.size()); - - List trainExamples = new ArrayList(); - List devExamples = new ArrayList(); - - for (int i = 0; i < split1; i++) - trainExamples.add(originalExamples.get(perm[i])); - for (int i = split2; i < originalExamples.size(); i++) - devExamples.add(originalExamples.get(perm[i])); - - return Pair.newPair(trainExamples, devExamples); - - } - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/EntityInstance.java b/src/edu/stanford/nlp/sempre/paraphrase/EntityInstance.java deleted file mode 100644 index 0da7160..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/EntityInstance.java +++ /dev/null @@ -1,30 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.SemType; - -import java.util.Map; - -public class EntityInstance { - - public final Context context; - public final String entity; - public final Interval interval; - public final Formula formula; - private Map featureVector; - public final SemType semType; - - public EntityInstance(Context context, String entity, Interval interval, - Formula formula, Map featureVector, SemType semType) { - this.context = context; - this.entity = entity; - this.interval = interval; - this.formula = formula; - this.featureVector = featureVector; - this.semType = semType; - } - - public String toString() { - return "context="+context+" entity="+entity + " formula="+formula+" features="+featureVector+" semType="+semType; - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/EntityModel.java b/src/edu/stanford/nlp/sempre/paraphrase/EntityModel.java deleted file mode 100644 index d2892e4..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/EntityModel.java +++ /dev/null @@ -1,104 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.Derivation; -import edu.stanford.nlp.sempre.Example; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.Parser; -import fig.basic.*; - -/** - * Provides a distribution over entity instances p(e,c,s|x) - * @author jonathanberant - * - */ -public abstract class EntityModel { - - public static class Options { - @Option public int verbose = 3; - } - public static Options opts = new Options(); - - final Parser parser; - Params params; - final ContextModel contextMapper; - - public EntityModel(Parser parser, ContextModel contextMapper) { - this.parser = parser; - this.contextMapper = contextMapper; - } - - public abstract TDoubleMap calcEntityInstanceDist(Example ex); -} - -class HeuristicEntityModel extends EntityModel { - - public HeuristicEntityModel(Parser parser,ContextModel contextMapper) { - super(parser,contextMapper); - params = new Params(); //no params in this model - } - - @Override - public TDoubleMap calcEntityInstanceDist(Example ex) { - - //First parse - StopWatchSet.begin("Parser.parse"); - parser.parse(params, ex, false); - StopWatchSet.end(); - //now compute distribution - if(ex.getPredDerivations().size()==0) - return new TDoubleMap(); - - LogInfo.begin_track("Generating entity distribtuion"); - Pair bestPair = null; - Derivation bestDeriv = null; - - for(Derivation currDeriv: ex.getPredDerivations()) { - Pair currPair = ParaphraseUtils.extractContextIntervalPair(currDeriv, ex); - if(bestPair==null || firstBetter(currPair,currDeriv,bestPair,bestDeriv,ex)) { - if(opts.verbose>=3 && bestDeriv != null) - LogInfo.logs("ParaphraseLearner: replacing context %s with formula %s for context %s with formula %s",bestPair,bestDeriv.formula,currPair,currDeriv.formula); - bestPair = currPair; - bestDeriv = currDeriv; - } - } - - TDoubleMap res = new TDoubleMap(); - Context bestContext = bestPair.getFirst(); - Interval bestInterval = bestPair.getSecond(); - EntityInstance entityInstance = new EntityInstance(bestContext, - ex.languageInfo.lemmaPhrase(bestInterval.start, bestInterval.end), - bestInterval, bestDeriv.formula, bestDeriv.getAllFeatureVector(),bestDeriv.type); - - if(opts.verbose>=3) - LogInfo.logs("ParaphraseLearner: generated entity instance=%s",entityInstance); - res.put(entityInstance, 1.0); - LogInfo.end_track(); - return res; - - - } - - private boolean firstBetter(Pair currPair, - Derivation currDeriv, Pair bestPair, - Derivation bestDeriv, Example ex) { - - if(contextMapper.containsKey(currPair.getFirst()) && - !contextMapper.containsKey(bestPair.getFirst())) - return true; - if(currPair.getSecond().properSuperset(bestPair.getSecond())) - return true; - if(currDeriv.getAllFeatureVector("basicStats :: entity.popularity") > - bestDeriv.getAllFeatureVector("basicStats :: entity.popularity")) - return true; - if(currDeriv.getAllFeatureVector("basicStats :: entity.popularity") < - bestDeriv.getAllFeatureVector("basicStats :: entity.popularity")) - return false; - if(currDeriv.getAllFeatureVector("tokensDistance :: entity.equal") > - bestDeriv.getAllFeatureVector("tokensDistance :: entity.equal")) - return true; - return false; - } - - - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarity.java b/src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarity.java deleted file mode 100644 index 8cb6b62..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarity.java +++ /dev/null @@ -1,46 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.Params; -import fig.basic.LispTree; - -public class FeatureSimilarity { - - public final double score; - public final FeatureVector featureVector; - public final String source; - public final String target; - - public FeatureSimilarity(FeatureVector fv, String source, String target, Params params) { - this.featureVector=fv; - this.source=source; - this.target=target; - score = fv.dotProduct(params); - } - - public FeatureSimilarity(FeatureVector fv, String source, String target, double score) { - this.featureVector=fv; - this.source=source; - this.target=target; - this.score=score; - } - - public FeatureSimilarity copy() { - FeatureVector newFv = new FeatureVector(); - newFv.add(featureVector); - return new FeatureSimilarity(newFv, source, target, score); - } - - public LispTree toLispTree() { - LispTree tree = LispTree.proto.newList(); - tree.addChild("similarity"); - tree.addChild(LispTree.proto.newList("source", source)); - tree.addChild(LispTree.proto.newList("target", target)); - tree.addChild(LispTree.proto.newList("sim_score", ""+score)); - return tree; - } - - public void clear() { - featureVector.clear(); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarityComputer.java b/src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarityComputer.java deleted file mode 100644 index cb90554..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarityComputer.java +++ /dev/null @@ -1,106 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.fbalignment.utils.MathUtils; -import fig.basic.Option; - -/** - * Interface for computing similarity from all kinds of surface features. - * @author jonathanberant - * - */ -public interface FeatureSimilarityComputer { - - public static class Options { - @Option(gloss="Whether to use the vsm model") public String mode; - } - public static Options opts = new Options(); - - public void computeSimilarity(ParaphraseExample ex, Params params); -} - -class FeatureSimilarityComputerFactory { - - public static FeatureSimilarityComputer getFeatureSimilarityComputer() { - if(FeatureSimilarityComputer.opts.mode.equals("lexical_overlap")) - return new LexicalOverlap(); - if(FeatureSimilarityComputer.opts.mode.equals("edit_distance")) - return new EditDistance(); - if(FeatureSimilarityComputer.opts.mode.equals("wan")) - return new WanFeatures(); - throw new RuntimeException("Illegal mode: " + FeatureSimilarityComputer.opts.mode); - } -} - -class LexicalOverlap implements FeatureSimilarityComputer { - @Override - public void computeSimilarity(ParaphraseExample ex, Params params) { - ex.ensureAnnotated(); - double jaccard = MathUtils.jaccard(ex.sourceInfo.lemmaTokens, ex.targetInfo.lemmaTokens); - FeatureVector fv = new FeatureVector(); - fv.add("Jaccard", "jaccard", jaccard); - ex.setVectorSpaceSimilarity(new FeatureSimilarity(fv,ex.source,ex.target,params)); - } - -} - -class EditDistance implements FeatureSimilarityComputer { - @Override - public void computeSimilarity(ParaphraseExample ex, Params params) { - ex.ensureAnnotated(); - int editDistance = ParaphraseUtils.editDistance(ex.sourceInfo.lemmaTokens, ex.targetInfo.lemmaTokens); - FeatureVector fv = new FeatureVector(); - fv.add("EditDistance", "distance", (double) editDistance); - ex.setVectorSpaceSimilarity(new FeatureSimilarity(fv,ex.source,ex.target,params)); - } -} - -class WanFeatures implements FeatureSimilarityComputer { - - @Override - public void computeSimilarity(ParaphraseExample ex, Params params) { - - ex.ensureAnnotated(); - - FeatureVector fv = new FeatureVector(); - double precision = MathUtils.coverage(ex.sourceInfo.tokens, ex.targetInfo.tokens); - double recall = MathUtils.coverage(ex.targetInfo.tokens, ex.sourceInfo.tokens); - double f1 = (precision+recall<=0d)? 0d : - (2 * precision * recall) / (precision+recall); - double lemmaPrecision = MathUtils.coverage(ex.sourceInfo.lemmaTokens, ex.targetInfo.lemmaTokens); - double lemmaRecall = MathUtils.coverage(ex.targetInfo.lemmaTokens, ex.sourceInfo.lemmaTokens); - addIfBetweenZeroAndOne(fv, "Wan", "precision", precision); - addIfBetweenZeroAndOne(fv, "Wan", "recall", recall); - addIfBetweenZeroAndOne(fv, "Wan", "f1", f1); - addIfBetweenZeroAndOne(fv, "Wan", "lemmaPrecision", lemmaPrecision); - addIfBetweenZeroAndOne(fv, "Wan", "lemmaRecall", lemmaRecall); - fv.add("Wan", "lengthDiff", ex.sourceInfo.numTokens()-ex.targetInfo.numTokens()); - fv.add("Wan", "AbsLengthDiff", Math.abs(ex.sourceInfo.numTokens()-ex.targetInfo.numTokens())); - int editDistance = ParaphraseUtils.editDistance(ex.sourceInfo.tokens, ex.targetInfo.tokens); - int lemmaEditDistance = ParaphraseUtils.editDistance(ex.sourceInfo.lemmaTokens, ex.targetInfo.lemmaTokens); - fv.add("Wan", "editDistance", editDistance); - fv.add("Wan", "lemmaEditDistance", lemmaEditDistance); - double precisionBleu = MathUtils.bleu(ex.sourceInfo.tokens, ex.targetInfo.tokens); - double recallBleu = MathUtils.bleu(ex.targetInfo.tokens, ex.sourceInfo.tokens); - double lemmaPecisionBleu = MathUtils.bleu(ex.sourceInfo.lemmaTokens, ex.targetInfo.lemmaTokens); - double lemmaRecallBleu = MathUtils.bleu(ex.targetInfo.lemmaTokens, ex.sourceInfo.lemmaTokens); - addIfBetweenZeroAndOne(fv, "Wan", "precisionBlue", precisionBleu); - addIfBetweenZeroAndOne(fv, "Wan", "recallBleu", recallBleu); - addIfBetweenZeroAndOne(fv, "Wan", "lemmaPecisionBleu", lemmaPecisionBleu); - addIfBetweenZeroAndOne(fv, "Wan", "lemmaRecallBleu", lemmaRecallBleu); - - ex.setVectorSpaceSimilarity(new FeatureSimilarity(fv,ex.source,ex.target,params)); - } - - public void addIfBetweenZeroAndOne(FeatureVector fv, String domain, String name, double value) { - if(value>1.0000000001) - throw new RuntimeException("Illegal feature value, feature="+name+", value="+value); - if(value<-0.0000000001) - throw new RuntimeException("Illegal feature value, feature="+name+", value="+value); - if(value>0.000000001) - fv.add(domain, name, value); - - } -} - diff --git a/src/edu/stanford/nlp/sempre/paraphrase/Interval.java b/src/edu/stanford/nlp/sempre/paraphrase/Interval.java deleted file mode 100644 index 0afef5a..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/Interval.java +++ /dev/null @@ -1,52 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -public class Interval { - public int start; - public int end; - - public Interval(int start, int end) { - this.start = start; - this.end = end; - } - - public void set(int start, int end) { - this.start = start; - this.end = end; - } - - public boolean superset(Interval other) { - return start<=other.start && end>=other.end; - } - - public boolean properSuperset(Interval other) { - return (start<=other.start && end>other.end) || - (start=other.end); - } - - public boolean subset(Interval other) { - return other.superset(this); - } - - public int length() { - return end-start; - } - - public double middle() { - return (((double) start+end)/2)+1; - } - - public boolean contains(int index) { - return index>=start && index> m = Maps.newHashMap(); - learn(m); - } - - private void learn(Map> evaluations) { - - LogInfo.begin_track("Learner.learn()"); - - // Averaged over all iterations - // Group -> evaluation for that group. - Map meanEvaluations = Maps.newHashMap(); - - // Clear - for (String group : dataset.groups()) - meanEvaluations.put(group, new Evaluation()); - - // Test and train - for (String group : dataset.groups()) { - Evaluation eval = processExamples(group,dataset.examples(group)); - MapUtils.addToList(evaluations, group, eval); - meanEvaluations.get(group).add(eval); - StopWatchSet.logStats(); - } - LogInfo.end_track(); - } - - private Evaluation processExamples(String group, List examples) { - - Evaluation evaluation = new Evaluation(); - if(group.equals("train")) { - learnContexts(group,examples,evaluation); - } - else { - infer(group,examples,evaluation); - } - return evaluation; - } - - private void learnContexts(String group, List examples, Evaluation eval) { - - if(opts.uploadContexts) { - LogInfo.log("Context map was uploaded so there is no context learning"); - contextModel.log(); - return; - } - if (examples.size() == 0) - return; - - final String prefix = group; - - Execution.putOutput("group", group); - LogInfo.begin_track_printAll( - "Processing %s: %s examples", prefix, examples.size()); - - for (int e = 0; e < examples.size(); e++) { - Example ex = examples.get(e); - - LogInfo.begin_track_printAll( - "%s: example %s/%s: %s", prefix, e, examples.size(), ex.id); - ex.log(); - Execution.putOutput("example", e); - - parseExample(ex, trainParser, true); - Map counts = new HashMap(); - computeExpectedCounts(ex, ex.getPredDerivations(), counts); - updateWeights(counts); - logExampleEvaluation(ex); - accumulateAndLogEvaluation(ex, eval, prefix); - - contextModel.map(ex); - ex.clearPredDerivations(); //to save memory since the beam size is so huge - LogInfo.end_track(); - } - logEvaluationStats(eval, prefix); - contextModel.normalize(); - contextModel.log(); - Json.writeValueHard(contextOut, contextModel); - String path = Execution.getFile("params"); - params.write(path); - - LogInfo.end_track(); - } - - private void infer(String group, List examples, Evaluation eval) { - if (examples.size() == 0) - return; - - final String prefix = group; - - Execution.putOutput("group", group); - LogInfo.begin_track_printAll( - "Processing %s: %s examples", prefix, examples.size()); - LogInfo.begin_track("Examples"); - - for (int e = 0; e < examples.size(); e++) { - Example ex = examples.get(e); - - LogInfo.begin_track_printAll( - "%s: example %s/%s: %s", prefix, e, examples.size(), ex.id); - ex.log(); - Execution.putOutput("example", e); - - //generate a distribution p(c,s,e|x) - now we will have a heuristic for choosing a single pair (c,e) - TDoubleMap entityInstanceDist = entityModel.calcEntityInstanceDist(ex); - //compute and sort the formulas - List predictions = contextModel.computePredictions(ex, entityInstanceDist); - //set evaluation - setEvaluation(ex, predictions); - logExampleEvaluation(ex); - accumulateAndLogEvaluation(ex, eval, prefix); - LogInfo.end_track(); - } - - LogInfo.end_track(); - logEvaluationStats(eval, prefix); - LogInfo.end_track(); - } - - public void setEvaluation(final Example ex, List predictions) { - final Evaluation eval = new Evaluation(); - int numCandidates = predictions.size(); - LogInfo.begin_track_printAll("ParaphraseLearner.setEvaluation: %d candidates", numCandidates); - - // Make sure at least the top derivation is executed. - for (Prediction prediction : predictions) { - prediction.ensureExecuted(executor); - } - - // Did we get the answer correct? - int correct_i = -1; // Index of first correct derivation - double maxCompatibility = 0.0; - double[] compatibilities = null; - if (ex.targetValue != null) { - compatibilities = new double[numCandidates]; - for (int i = 0; i < numCandidates; i++) { - Prediction prediction = predictions.get(i); - compatibilities[i] = prediction.compatibility = ex.targetValue.getCompatibility(prediction.value); - - // Must be fully compatible to count as correct. - if (compatibilities[i] == 1 && correct_i == -1) - correct_i = i; - //record maximum compatibility for partial oracle - maxCompatibility = Math.max(compatibilities[i], maxCompatibility); - } - } - - // Number of scored formulas which have the same top score - int numTop = 0; - double topMass = 0; - if (ex.targetValue != null) { - while (numTop < numCandidates && - compatibilities[numTop] > 0.0d && - Math.abs(predictions.get(numTop).score - predictions.get(0).score) < 1e-10) { - topMass += predictions.get(numTop).score; - numTop++; - } - } - double correct = 0; - double partial_correct = 0; - if (ex.targetValue != null) { - for (int i = 0; i < numTop; i++) { - if (compatibilities[i] == 1) correct += predictions.get(i).score / topMass; - if (compatibilities[i] > 0) - partial_correct += (compatibilities[i] * predictions.get(i).score) / topMass; - } - } - - // Fully correct - for (int i = 0; i < predictions.size(); i++) { - Prediction prediction = predictions.get(i); - if (compatibilities != null && compatibilities[i] == 1) { - LogInfo.logs( - "True@%04d: %s [score=%s] value=%s compatibility=%s", i, prediction.formula, - Fmt.D(prediction.score),prediction.value,prediction.compatibility); - } - } - // Partially correct - for (int i = 0; i < predictions.size(); i++) { - Prediction prediction = predictions.get(i); - if (compatibilities != null && compatibilities[i] > 0 && compatibilities[i] < 1) { - LogInfo.logs( - "Part@%04d: %s [score=%s] value=%s compatibility=%s", i, prediction.formula, - Fmt.D(prediction.score),prediction.value,prediction.compatibility); - } - } - // Anything that's predicted. - for (int i = 0; i < predictions.size(); i++) { - Prediction prediction = predictions.get(i); - // Either print all predictions or this prediction is worse by some amount. - boolean print = i < 50; - if (print) { - LogInfo.logs( - "Pred@%04d: %s [score=%s] value=%s compatibility=%s", i, prediction.formula, - Fmt.D(prediction.score),prediction.value,prediction.compatibility); - } - } - - eval.add("correct", correct); - eval.add("oracle", correct_i != -1); - eval.add("partCorrect", partial_correct); - eval.add("partOracle", maxCompatibility); - - eval.add("numCandidates", numCandidates); // From this parse - if (predictions.size() > 0) - eval.add("parsedNumCandidates", predictions.size()); - - // Finally, set all of these stats as the example's evaluation. - ex.setEvaluation(eval); - LogInfo.end_track(); - } - - - private void parseExample(Example ex, Parser parser, boolean execAllowed) { - StopWatchSet.begin("Parser.parse"); - parser.parse(params, ex, execAllowed); - StopWatchSet.end(); - } - - private void logExampleEvaluation(Example ex) { - LogInfo.logs("Current: %s", ex.computeTotalEvaluation().summary()); - } - - private void accumulateAndLogEvaluation(Example ex, - Evaluation evaluation, - String prefix) { - evaluation.add(ex.computeTotalEvaluation()); - LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary()); - } - - private void logEvaluationStats(Evaluation evaluation, String prefix) { - LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary()); - evaluation.add(LexiconFn.lexEval); - evaluation.logStats(prefix); - evaluation.putOutput(prefix); - } - - private void computeExpectedCounts(Example ex, List derivations, Map counts) { - - double[] trueScores; - double[] predScores; - - int n = derivations.size(); - if (n == 0) - return; - - trueScores = new double[n]; - predScores = new double[n]; - for (int i = 0; i < n; i++) { - Derivation deriv = derivations.get(i); - double r = reward(ex.targetValue, deriv.getValue()); - trueScores[i] = deriv.getScore() + r; - predScores[i] = deriv.getScore(); - - } - - if (!NumUtils.expNormalize(trueScores)) return; - if (!NumUtils.expNormalize(predScores)) return; - for (int i = 0; i < n; i++) { - Derivation deriv = derivations.get(i); - double incr = trueScores[i] - predScores[i]; - deriv.incrementAllFeatureVector(incr, counts); - } - } - - private double reward(Value targetValue, Value predValue) { - if (targetValue != null) { - return Math.log(targetValue.getCompatibility(predValue)); - } - return 0; - } - - private void updateWeights(Map counts) { - StopWatchSet.begin("Learner.updateWeights"); - LogInfo.begin_track("Updating weights"); - double sum = 0; - for (double v : counts.values()) sum += v * v; - LogInfo.logs("L2 norm: %s", Math.sqrt(sum)); - params.update(counts); - counts.clear(); - LogInfo.end_track(); - StopWatchSet.end(); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/NnBuilder.java b/src/edu/stanford/nlp/sempre/paraphrase/NnBuilder.java deleted file mode 100644 index cc139ac..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/NnBuilder.java +++ /dev/null @@ -1,66 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.*; -import fig.basic.Option; -import fig.basic.Utils; - -import java.util.List; - -/** - * Basic components for transformation learning - * @author jonathanberant - */ -public class NnBuilder { - public static class Options { - @Option public String packageName = "edu.stanford.nlp.sempre"; - @Option public String executor = "SparqlExecutor"; - @Option public List trainGrammarPath; - @Option public List testGrammarPath; - } - - public static Options opts = new Options(); - - public Executor executor; - public FeatureExtractor extractor; - public Parser trainParser; - public Parser testParser; - public ContextModel contextMapper; - - public void build() { - executor = null; - extractor = null; - trainParser = null; - contextMapper = null; - buildUnspecified(); - } - - public void buildUnspecified() { - - // Train grammar - Grammar.opts.inPaths = opts.trainGrammarPath; - Grammar trainGrammar = new Grammar(); - trainGrammar.read(); - - //Test grammar - Grammar.opts.inPaths = opts.testGrammarPath; - Grammar testGrammar = new Grammar(); - testGrammar.read(); - - // Executor - if (executor == null) { - executor = (Executor) Utils.newInstanceHard(opts.packageName + "." + opts.executor); - } - - // Feature extractors - if (extractor == null) - extractor = new FeatureExtractor(executor); - - // Train Parser - if (trainParser == null) - trainParser = new BeamParser(trainGrammar, extractor, executor); - - // Test parser - if(testParser == null) - testParser = new BeamParser(testGrammar, extractor, executor); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseBuilder.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseBuilder.java deleted file mode 100644 index ad382eb..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseBuilder.java +++ /dev/null @@ -1,34 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import com.google.common.base.Strings; - -import edu.stanford.nlp.sempre.Executor; -import edu.stanford.nlp.sempre.Params; -import fig.basic.Option; -import fig.basic.Utils; - -public class ParaphraseBuilder { - public static class Options { - @Option public String inParamsPath; - @Option public String packageName = "edu.stanford.nlp.sempre"; - @Option public String executor = "SparqlExecutor"; - } - public static Options opts = new Options(); - - public Params params; - public Executor executor; - - public void build() { - // Parameters - if (params == null) { - params = new Params(); - if (!Strings.isNullOrEmpty(opts.inParamsPath)) - params.read(opts.inParamsPath); - } - - // Executor - if (executor == null) { - executor = (Executor) Utils.newInstanceHard(opts.packageName + "." + opts.executor); - } - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDataset.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDataset.java deleted file mode 100644 index 56532bf..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDataset.java +++ /dev/null @@ -1,254 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.ArrayList; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Lists; - -import edu.stanford.nlp.sempre.Json; -import fig.basic.IOUtils; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Option; -import fig.basic.Pair; -import fig.exec.Execution; - -/** - * Holds both paraphrasing examples and parsing examples - * TODO - now there is duplicate code for paraphrasing and parsing examples - * @author jonathanberant - * - */ -public class ParaphraseDataset { - - public static class Options { - @Option(gloss = "Paths to read paraphrase example files (format: :)") - public ArrayList> paraphraseInPaths = new ArrayList<>(); - @Option(gloss = "Paths to read parsing example files (format: :)") - public ArrayList> parsingInPaths = new ArrayList<>(); - - - @Option(gloss = "Maximum number of paraphrase examples to read") - public ArrayList> maxParaphraseExamples = new ArrayList<>(); - @Option(gloss = "Maximum number of parsing examples to read") - public ArrayList> maxParsingExamples = new ArrayList<>(); - - // Training file gets split into: - // | trainFrac --> | | <-- devFrac | - @Option(gloss = "Fraction of trainExamples (from the beginning) to keep for training") - public double trainFrac = 1; - @Option(gloss = "Fraction of trainExamples (from the end) to keep for development") - public double devFrac = 0; - @Option(gloss = "Used to randomly divide training examples") - public Random splitRandom = new Random(1); - } - public static Options opts = new Options(); - - // Group id -> examples in that group - private LinkedHashMap> paraphraseExamples = new LinkedHashMap<>(); - private LinkedHashMap> parsingExamples = new LinkedHashMap<>(); - - public Set parsingGroups() { return parsingExamples.keySet(); } - public List paraphraseExamples(String group) { return paraphraseExamples.get(group); } - public List parsingExamples(String group) { return parsingExamples.get(group); } - - /** For JSON. */ - static class ParaphrasingGroupInfo { - @JsonProperty final String group; - @JsonProperty final List examples; - String path; // Optional, used if this was read from a path. - @JsonCreator - public ParaphrasingGroupInfo(@JsonProperty("group") String group, - @JsonProperty("examples") List examples) { - this.group = group; - this.examples = examples; - } - } - - static class ParsingGroupInfo { - @JsonProperty final String group; - @JsonProperty final List examples; - String path; // Optional, used if this was read from a path. - @JsonCreator - public ParsingGroupInfo(@JsonProperty("group") String group, - @JsonProperty("examples") List examples) { - this.group = group; - this.examples = examples; - } - } - - /** For JSON. */ - @JsonProperty("paraphrasing_groups") - public List getAllParaphrasingGroupInfos() { - List all = Lists.newArrayList(); - for (Map.Entry> entry : paraphraseExamples.entrySet()) - all.add(new ParaphrasingGroupInfo(entry.getKey(), entry.getValue())); - return all; - } - - /** For JSON. */ - @JsonProperty("parsing_groups") - public List getAllParsingGroupInfos() { - List all = Lists.newArrayList(); - for (Map.Entry> entry : parsingExamples.entrySet()) - all.add(new ParsingGroupInfo(entry.getKey(), entry.getValue())); - return all; - } - - /** For JSON. */ - @JsonCreator - public static ParaphraseDataset fromGroupInfos(@JsonProperty("paraphrasing_groups") List paraphrasingGroups, - @JsonProperty("parsing_groups") List parsingGroups) { - ParaphraseDataset d = new ParaphraseDataset(); - d.readFromParaphrasingGroupInfos(paraphrasingGroups); - d.readFromParsingGroupInfos(parsingGroups); - return d; - } - - public void read() { - readFromParaphrasingPathPairs(opts.paraphraseInPaths); - readFromParsingPathPairs(opts.parsingInPaths); - collectStats(); - } - - public void readFromParaphrasingPathPairs(List> pathPairs) { - - List groups = Lists.newArrayListWithCapacity(pathPairs.size()); - for (Pair pathPair : pathPairs) { - String group = pathPair.getFirst(); - String path = pathPair.getSecond(); - List examples = Json.readValueHard( - IOUtils.openInHard(path), - new TypeReference>() {}); - ParaphrasingGroupInfo gi = new ParaphrasingGroupInfo(group, examples); - gi.path = path; - groups.add(gi); - } - readFromParaphrasingGroupInfos(groups); - } - - public void readFromParsingPathPairs(List> pathPairs) { - - List groups = Lists.newArrayListWithCapacity(pathPairs.size()); - for (Pair pathPair : pathPairs) { - String group = pathPair.getFirst(); - String path = pathPair.getSecond(); - List examples = Json.readValueHard( - IOUtils.openInHard(path), - new TypeReference>() {}); - ParsingGroupInfo gi = new ParsingGroupInfo(group, examples); - gi.path = path; - groups.add(gi); - } - readFromParsingGroupInfos(groups); - } - - private void readFromParaphrasingGroupInfos(List paraphrasingGroupInfos) { - LogInfo.begin_track_printAll("Dataset.read paraphrasing"); - - for (ParaphrasingGroupInfo paraphrasingGroupInfo : paraphrasingGroupInfos) { - int maxExamples = getMaxExamplesForParaphrasingGroup(paraphrasingGroupInfo.group); - List examples = paraphraseExamples.get(paraphrasingGroupInfo.group); - if (examples == null) - paraphraseExamples.put(paraphrasingGroupInfo.group, examples = new ArrayList<>()); - readParaphrasingHelper(paraphrasingGroupInfo.examples, maxExamples, examples, paraphrasingGroupInfo.path); - } - Pair,List> splitExamples = - DatasetUtils.splitTrainFromDev(MapUtils.get(paraphraseExamples, "train", new ArrayList()), - opts.trainFrac,opts.devFrac, - opts.splitRandom); - paraphraseExamples.put("train", splitExamples.getFirst()); - paraphraseExamples.put("dev", splitExamples.getSecond()); - LogInfo.end_track(); - } - - private void readFromParsingGroupInfos(List parsingGroupInfo) { - LogInfo.begin_track_printAll("Dataset.read parsing"); - - for (ParsingGroupInfo groupInfo : parsingGroupInfo) { - int maxExamples = getMaxExamplesForParsingGroup(groupInfo.group); - List examples = parsingExamples.get(groupInfo.group); - if (examples == null) - parsingExamples.put(groupInfo.group, examples = new ArrayList<>()); - readParsingHelper(groupInfo.examples, maxExamples, examples, groupInfo.path); - } - - Pair,List> splitExamples = - DatasetUtils.splitTrainFromDev(MapUtils.get(parsingExamples, "train", new ArrayList()), - opts.trainFrac, - opts.devFrac, - opts.splitRandom); - parsingExamples.put("train", splitExamples.getFirst()); - parsingExamples.put("dev", splitExamples.getSecond()); LogInfo.end_track(); - } - - private void readParaphrasingHelper(List incoming, - int maxExamples, - List examples, - String path) { - if (examples.size() >= maxExamples) - return; - - int i = 0; - for (ParaphraseExample ex : incoming) { - ex.id=""+ (i++); - if (examples.size() >= maxExamples) break; - LogInfo.logs("Example %s: source=%s target=%s => %s", - ex.id, ex.source, ex.target, ex.goldValue); - examples.add(ex); - } - } - - private void readParsingHelper(List incoming, - int maxExamples, - List examples, - String path) { - if (examples.size() >= maxExamples) - return; - - int i = 0; - for (ParsingExample ex : incoming) { - if (examples.size() >= maxExamples) break; - - String id = ""+i; - ex = new ParsingExample.Builder().withExample(ex).setId(id).createExample(); - - i++; - ex.preprocess(); - LogInfo.logs("Example %s (%d): %s => %s", - ex.id, examples.size(), ex.getTokens(), ex.targetValue); - examples.add(ex); - } - } - - private static int getMaxExamplesForParaphrasingGroup(String group) { - int maxExamples = Integer.MAX_VALUE; - for (Pair maxPair : opts.maxParaphraseExamples) - if (maxPair.getFirst().equals(group)) - maxExamples = maxPair.getSecond(); - return maxExamples; - } - - private static int getMaxExamplesForParsingGroup(String group) { - int maxExamples = Integer.MAX_VALUE; - for (Pair maxPair : opts.maxParsingExamples) - if (maxPair.getFirst().equals(group)) - maxExamples = maxPair.getSecond(); - return maxExamples; - } - - private void collectStats() { - LogInfo.begin_track_printAll("Dataset stats"); - for (Map.Entry> e : parsingExamples.entrySet()) - Execution.putLogRec("numExamples." + e.getKey(), e.getValue().size()); - LogInfo.end_track(); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDerivation.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDerivation.java deleted file mode 100644 index f57bddb..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDerivation.java +++ /dev/null @@ -1,137 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -import edu.stanford.nlp.sempre.Evaluation; -import edu.stanford.nlp.sempre.Executor; -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.FormulaGenerationInfo; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.Value; -import edu.stanford.nlp.sempre.paraphrase.Aligner.Alignment; -import fig.basic.LispTree; -import fig.basic.NumUtils; -import fig.basic.StopWatchSet; - -/** - * Derivation paraphrase, canonical utterance and logical form - */ -public class ParaphraseDerivation { - - public final LanguageInfo langInfo; - public final FeatureSimilarity vsSimilarity; - public final Alignment alignment; - - public final Formula formula; - public final FormulaGenerationInfo fgInfo; - - FeatureVector featureVector; - double score = Double.NaN; - double prob = Double.NaN; - Value value; - Evaluation executorStats; - double compatibility = Double.NaN; - - public ParaphraseDerivation(LanguageInfo lInfo, - ParaphraseExample paraEx, FormulaGenerationInfo fgInfo, - ParaphraseFeatureExtractor fExtractor, Params params) { - this.langInfo = lInfo; - this.vsSimilarity=paraEx.featureSimilarity; - this.alignment=paraEx.alignment; - this.fgInfo=fgInfo; - this.formula = fgInfo.generateFormula(); - //extract feature vector - featureVector = new FeatureVector(); - fExtractor.extractParaphraseDerivationFeatures(this); - //compute score - we can do this here since all features are extracted in the constructor - scoreProofDeriv(params); - } - - public void ensureExecuted(Executor executor) { - if (!isExecuted()) { - StopWatchSet.begin("Executor.execute"); - Executor.Response response = executor.execute(formula); - StopWatchSet.end(); - value = response.value; - executorStats = response.stats; - } - } - - public boolean isExecuted() { return value != null; } - - //Generate a probability distribution over derivations given their scores. - public static double[] getProbs(List derivations, double temperature) { - double[] probs = new double[derivations.size()]; - for (int i = 0; i < derivations.size(); i++) - probs[i] = derivations.get(i).score / temperature; - if (probs.length > 0) - NumUtils.expNormalize(probs); - return probs; - } - - public String toString() { return toLispTree().toString(); } - - private LispTree toLispTree() { - LispTree tree = LispTree.proto.newList(); - tree.addChild("derivation"); - - if (formula != null) - tree.addChild(LispTree.proto.newList("formula", formula.toLispTree())); - if (value != null) { - tree.addChild(LispTree.proto.newList("value", value.toLispTree())); - } - if(vsSimilarity!=null) - tree.addChild(vsSimilarity.toLispTree()); - if(alignment!=null) - tree.addChild(alignment.toLispTree()); - return tree; - } - - public void incrementAllFeatureVector(double incr, Map counts) { - //increment all feature components - featureVector.increment(incr, counts); - if(vsSimilarity!=null) - vsSimilarity.featureVector.increment(incr, counts); - if(alignment!=null) - alignment.featureVector.increment(incr, counts); - } - - private void scoreProofDeriv(Params params) { - - double vsmScore = vsSimilarity!=null ? vsSimilarity.featureVector.dotProduct(params) : 0.0; - double alignmentScore = alignment!=null ? alignment.featureVector.dotProduct(params) :0.0; - double derivationScore = featureVector.dotProduct(params); - if(vsmScore==Double.NaN || - alignmentScore==Double.NaN || - derivationScore==Double.NaN - ) - throw new RuntimeException("One of the scores is not a number"); - score = - vsmScore+alignmentScore+derivationScore; - } - - public void clear() { - if(vsSimilarity!=null) - vsSimilarity.clear(); - if(alignment!=null) - alignment.clear(); - featureVector.clear(); - } - - public static class ParaphraseDerivationComparator implements Comparator { - @Override - public int compare(ParaphraseDerivation o1, ParaphraseDerivation o2) { - if(o1.score>o2.score) - return -1; - if(o2.score>o1.score) - return 1; - return 0; - } - } - - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseExample.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseExample.java deleted file mode 100644 index 31ec1bc..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseExample.java +++ /dev/null @@ -1,165 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.*; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import edu.stanford.nlp.sempre.*; -import edu.stanford.nlp.sempre.paraphrase.Aligner.Alignment; -import edu.stanford.nlp.sempre.paraphrase.rules.ParaphraseAlignment; -import edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosRule; -import edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosSequence; -import edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosSequence.LemmaAndPos; -import fig.basic.Fmt; -import fig.basic.LogInfo; -import fig.basic.MemUsage; - -public class ParaphraseExample { - - public String id=null; - @JsonProperty public final String source; - @JsonProperty public final String target; - @JsonProperty private Formula formula; //formula from which the paraphrase was generated - @JsonProperty Value goldValue; - - public LanguageInfo sourceInfo; - public LanguageInfo targetInfo; - - FeatureSimilarity featureSimilarity; - Alignment alignment; - - Evaluation eval = new Evaluation(); - - private static Map annotationCache = new HashMap<>(); - - public ParaphraseExample(String source, String target, BooleanValue value) { - this.source = source; - this.target = target; - this.goldValue = value; - synchronized(annotationCache) { - this.sourceInfo=annotationCache.get(source); //null if not there - this.targetInfo=annotationCache.get(target); //null if not there - } - } - - @JsonCreator - public ParaphraseExample(@JsonProperty("source") String source, @JsonProperty("target") String target, - @JsonProperty("formula") Formula f, @JsonProperty("value") BooleanValue value) { - this.source = source; - this.target = target; - this.formula = f; - this.goldValue = value; - } - - //IF WE HAVE A NULL PROOF IT SEEMS THAT THIS IS BROKEN - public double computeExampleScore() { - throw new RuntimeException("This method might be broken because there is now a null proof that" + - "can on the predicted proofs"); - // if(predictedProofs==null) - // throw new RuntimeException("Transformation was not performed. source: " + source + " target; " + target); - // if(predictedProofs.size()==0) - // return 0.0; - // if(predictedProofs.get(0).score<0.0) - // throw new RuntimeException("Supporting only positive scores for proofs"); - // return 1.0; - } - - - /** - * Aligns from left to right without any crossing alignments - */ - public ParaphraseAlignment align() { - - ensureAnnotated(); - int[] sourceAlignment = new int[sourceInfo.tokens.size()]; - int[] targetAlignment = new int[targetInfo.tokens.size()]; - Arrays.fill(sourceAlignment, -1); - Arrays.fill(targetAlignment, -1); - - int lastAlignedIndex = -1; - for(int i = 0; i < sourceInfo.tokens.size();++i) { - for(int j = lastAlignedIndex+1; j < targetInfo.tokens.size(); ++j) { - if(sourceInfo.lemmaTokens.get(i).equals(targetInfo.lemmaTokens.get(j))) { - lastAlignedIndex=sourceAlignment[i]=j; - targetAlignment[j]=i; - break; - } - } - } - return new ParaphraseAlignment(sourceAlignment,targetAlignment); - } - - public void ensureAnnotated() { - if(sourceInfo==null) { - sourceInfo = new LanguageInfo(); - sourceInfo.analyze(this.source); - synchronized (annotationCache) { - annotationCache.put(source, sourceInfo); - } - } - if(targetInfo==null) { - this.targetInfo = new LanguageInfo(); - this.targetInfo.analyze(this.target); - synchronized (annotationCache) { - annotationCache.put(source, sourceInfo); - } - } - } - - public String toJson() { - return Json.writeValueAsStringHard(this); - } - - public LemmaPosRule getRule(Interval sourceInterval, Interval targetInterval) { - return new LemmaPosRule(computeTemplate(sourceInfo, sourceInterval), - computeTemplate(targetInfo, targetInterval)); - } - - public LemmaPosSequence computeTemplate(LanguageInfo info, Interval interval) { - - List res = new ArrayList<>(); - for(int i = interval.start; i < interval.end; ++i) { - res.add(new LemmaAndPos(info.lemmaTokens.get(i),info.posTags.get(i))); - } - return new LemmaPosSequence(res); - } - - - public void setVectorSpaceSimilarity(FeatureSimilarity vsSimilarity) { - this.featureSimilarity=vsSimilarity; - } - - public void setAlignment(Alignment alignment) { - this.alignment=alignment; - } - - - public void log() { - ensureAnnotated(); - LogInfo.begin_track_printAll("Example"); - LogInfo.log("Id: " + id); - LogInfo.log("Source: " + source); - LogInfo.logs("Source lemmas: %s", sourceInfo.lemmaTokens); - LogInfo.logs("Source POS tags: %s", sourceInfo.posTags); - LogInfo.logs("Source NER tags: %s", sourceInfo.nerTags); - LogInfo.log("Target: " + target); - LogInfo.logs("Target lemmas: %s", targetInfo.lemmaTokens); - LogInfo.logs("Target POS tags: %s", targetInfo.posTags); - LogInfo.logs("Target NER tags: %s", targetInfo.nerTags); - LogInfo.log("Value: " + goldValue); - LogInfo.end_track(); - } - - public void setEvaluation(Params params) { - setEvaluation(params,false); - } - - public void setEvaluation(Params params, boolean print) { - } - - - public static long cacheSize() { - return MemUsage.getBytes(annotationCache); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureExtractor.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureExtractor.java deleted file mode 100644 index dd1b6a1..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureExtractor.java +++ /dev/null @@ -1,85 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.ErrorValue; -import edu.stanford.nlp.sempre.Executor; -import edu.stanford.nlp.sempre.FormulaGenerationInfo; -import edu.stanford.nlp.sempre.Formulas; -import edu.stanford.nlp.sempre.FreebaseInfo; -import edu.stanford.nlp.sempre.ListValue; -import edu.stanford.nlp.sempre.NumberValue; - -public class ParaphraseFeatureExtractor { - - private Executor executor; - - public ParaphraseFeatureExtractor(Executor executor) { - this.executor = executor; - } - - public void extractParaphraseDerivationFeatures(ParaphraseDerivation pDerivation) { - extractionDenotationFeatures(pDerivation); - extractFormulaFeatures(pDerivation); - extractWhTypeFeature(pDerivation); - extractNamedEntityFeature(pDerivation); - } - - private void extractNamedEntityFeature(ParaphraseDerivation pDerivation) { - if(!ParaphraseFeatureMatcher.containsDomain("NamedEntity")) return; - String namedEntity = pDerivation.langInfo.nerTags.get(pDerivation.fgInfo.entityInfo1.span.first); - pDerivation.featureVector.add("NamedEntity",namedEntity); - } - - private void extractWhTypeFeature(ParaphraseDerivation pDerivation) { - - if(!ParaphraseFeatureMatcher.containsDomain("WhType")) return; - - if(pDerivation.langInfo.posTags.get(0).startsWith("W")) { - pDerivation.featureVector.add("WhType", - "token0="+pDerivation.langInfo.tokens.get(0)+","+ - "type="+FreebaseInfo.getSingleton().coarseType(pDerivation.fgInfo.bInfo.expectedType1)); - } - } - - private void extractFormulaFeatures(ParaphraseDerivation pDerivation) { - if (!ParaphraseFeatureMatcher.containsDomain("Formula")) return; - FormulaGenerationInfo fgInfo = pDerivation.fgInfo; - pDerivation.featureVector.add("Formula", "binPopularity",Math.log(fgInfo.bInfo.popularity+1)); - pDerivation.featureVector.add("Formula", "entPopularity",Math.log(pDerivation.fgInfo.entityInfo1.popularity+1)); - pDerivation.featureVector.add("Formula", "binary="+fgInfo.bInfo.formula); - if(fgInfo.isUnary) { - pDerivation.featureVector.add("Formula", "uPopularity",Math.log(fgInfo.uInfo.popularity+1)); - pDerivation.featureVector.add("Formula", "unary"); - } - if(fgInfo.isInject) { - pDerivation.featureVector.add("Formula", "entPopularity",Math.log(pDerivation.fgInfo.entityInfo2.popularity+1)); - pDerivation.featureVector.add("Formula", "injected="+fgInfo.injectedInfo.formula); - pDerivation.featureVector.add("Formula", "injectedType="+fgInfo.injectedInfo.expectedType2); - pDerivation.featureVector.add("Formula", "inject"); - } - } - - private void extractionDenotationFeatures(ParaphraseDerivation pDerivation) { - if (!ParaphraseFeatureMatcher.containsDomain("Denotation")) return; - pDerivation.ensureExecuted(executor); - if(pDerivation.value instanceof ErrorValue) { - pDerivation.featureVector.add("Denotation", "error"); - return; - } - - if (!(pDerivation.value instanceof ListValue)) - throw new RuntimeException("Derivation value is not a list: " + pDerivation.value); - - ListValue list = (ListValue) pDerivation.value; - if (Formulas.isCountFormula(pDerivation.formula)) { - if (list.values.size() != 1) { - throw new RuntimeException( - "Evaluation of count formula " + pDerivation.formula + " has size " + list.values.size()); - } - int count = (int)((NumberValue)list.values.get(0)).value; - pDerivation.featureVector.add("Denotation", "count-size" + (count == 0 ? "=0" : ">0")); - } else { - int size = list.values.size(); - pDerivation.featureVector.add("Denotation", "size" + (size < 3 ? "=" + size : ">=" + 3)); - } - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureMatcher.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureMatcher.java deleted file mode 100644 index 0b138d9..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureMatcher.java +++ /dev/null @@ -1,31 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.HashSet; -import java.util.Set; - -import edu.stanford.nlp.sempre.FeatureMatcher; -import fig.basic.Option; - -public class ParaphraseFeatureMatcher implements FeatureMatcher { - - public static class Options { - @Option(gloss = "feature domains we allow") - public Set featureDomains = new HashSet(); - @Option(gloss = "constraints on features") - public Set featureConstraints = new HashSet(); - } - public static Options opts = new Options(); - - public static boolean containsDomain(String domain) { - return opts.featureDomains.contains(domain); - } - - @Override - public boolean matches(String feature) { - if(opts.featureConstraints.contains("unlex")) { - if(feature.contains("tokens=") || feature.contains("lemmas=")) - return false; - } - return true; - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseLearner.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseLearner.java deleted file mode 100644 index f886d05..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseLearner.java +++ /dev/null @@ -1,130 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.io.IOException; -import java.util.List; - - -import edu.stanford.nlp.sempre.Evaluation; -import edu.stanford.nlp.sempre.Executor; -import edu.stanford.nlp.sempre.Params; -import fig.basic.LogInfo; -import fig.basic.Option; -import fig.basic.Parallelizer; -import fig.basic.StopWatchSet; -import fig.basic.Utils; -import fig.exec.Execution; - -/** - * Input: Two sets of examples (1) paraphrase data set - positive and negative examples - * (2) a semantic parsing data set - question-answer pairs - * Output: A model for paraphrase that can be used to judge whether a pair of utternaces are paraphrases - * and also can be used for semantic parsing by paraphrasing a target question to a question that was generated - * from a database formula - * @author jonathanberant - * - */ -public class ParaphraseLearner { - - public static class Options { - @Option(gloss = "Number of iterations to train") - public int maxTrainIters = 0; - @Option(gloss = "Whether to train or only to infer") - public boolean inferOnly; - @Option(gloss = "Number of threads") - public int numOfThreads=8; - @Option(gloss = "Whether to use binary logstic regression updates") - public boolean binaryLogistic=false; - @Option(gloss = "Whether to update based on partial reward.") - public boolean partialReward = true; - } - public static Options opts = new Options(); - - //paraphrasing components - private final Params params; - private final ParaphraseDataset paraphraseDataset; - private final ParaphraseParser paraParser; - - - public ParaphraseLearner(Params p, ParaphraseDataset pDataset, Executor executor) throws IOException { - this.params=p; - this.paraphraseDataset=pDataset; - this.paraParser = new ParaphraseParser(executor); - } - - public void learn() { - learn(-1); - } - - public void learn(int iters) { - LogInfo.begin_track("Learner.learn()"); - - if (iters < 0) - iters = opts.maxTrainIters; - - for (int iter = 0; iter <= iters; iter++) { - - if(iter == iters && opts.inferOnly) continue; //when only inferring no need to go over training set again - LogInfo.begin_track("Iteration %s/%s", iter, iters); - Execution.putOutput("iter", iter); - - LogInfo.begin_track("Learn from parsing dataset"); - for (String group : paraphraseDataset.parsingGroups()) { - boolean lastIter = iter == iters; - boolean updateWeights = group.equals("train") && !lastIter && !opts.inferOnly; // Don't train on last iteration - processParsingExamples( - iter, - group, - paraphraseDataset.parsingExamples(group), - updateWeights); - StopWatchSet.logStats(); - } - LogInfo.end_track(); - - // Write out parameters - String path = Execution.getFile("params." + iter); - if (path != null) { - params.write(path); - Utils.systemHard("ln -sf params." + iter + " " + Execution.getFile("params")); - } - LogInfo.end_track(); - } - - LogInfo.end_track(); - } - - private Evaluation processParsingExamples(int iter, String group, - List parsingExamples, boolean updateWeights) { - - Evaluation totalEval = new Evaluation(); - - if (parsingExamples.size() == 0) - return totalEval; - - final String prefix = "parsing_iter=" + iter + "." + group; - Execution.putOutput("group", group); - - LogInfo.begin_track_printAll( - "Processing %s: %s examples", prefix, parsingExamples.size()); - LogInfo.begin_track("Examples"); - - - Parallelizer paral = new Parallelizer<>(opts.numOfThreads); - ParsingExampleProcessor processor = new ParsingExampleProcessor(paraParser, params, prefix, updateWeights, totalEval); - LogInfo.begin_threads(); - paral.process(parsingExamples, processor); - LogInfo.end_threads(); - - params.finalizeWeights(); - - LogInfo.end_track(); - logEvaluationStats(totalEval, prefix); - LogInfo.end_track(); - return totalEval; - } - - private void logEvaluationStats(Evaluation evaluation, String prefix) { - LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary()); - evaluation.logStats(prefix); - evaluation.putOutput(prefix); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseMain.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseMain.java deleted file mode 100644 index 2bcfa85..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseMain.java +++ /dev/null @@ -1,92 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.*; -import edu.stanford.nlp.sempre.fbalignment.lexicons.EntityLexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.WordDistance; -import edu.stanford.nlp.sempre.paraphrase.paralex.ParalexRules; -import edu.stanford.nlp.sempre.paraphrase.paralex.PhraseTable; -import edu.stanford.nlp.sempre.paraphrase.rules.RuleApplier; -import edu.stanford.nlp.sempre.paraphrase.rules.Rulebase; -import edu.stanford.nlp.sempre.paraphrase.rules.SubstitutionRuleExtractor; -import edu.stanford.nlp.sempre.paraphrase.rules.SyntacticRuleSet; -import edu.stanford.nlp.sempre.paraphrase.rules.VerbSemClassExtractor; -import fig.basic.LogInfo; -import fig.basic.Option; -import fig.basic.OptionsParser; -import fig.exec.Execution; - -import java.io.IOException; - -public class ParaphraseMain implements Runnable { - - @Option (gloss="run mode") public String mode="train"; - @Option (gloss="whether to visualize contexts to formulas") public boolean visualize=false; - - public void run() { - - LogInfo.logs("run mode: %s",mode); - if(mode.equals("train")) { - ParaphraseBuilder tBuilder = new ParaphraseBuilder(); - tBuilder.build(); - ParaphraseDataset paraphraseDataset = new ParaphraseDataset(); - paraphraseDataset.read(); - try { - ParaphraseLearner paraphraseLearner = new ParaphraseLearner(tBuilder.params, paraphraseDataset, tBuilder.executor); - paraphraseLearner.learn(); - } catch(IOException e) { - throw new RuntimeException(e); - } - } - else throw new RuntimeException("Unknown mode: " + mode); - } - - public static void main(String[] args) { - - OptionsParser parser = new OptionsParser(); - parser.registerAll( - new Object[]{ - "Master", Master.opts, - "ParaphraseBuilder", ParaphraseBuilder.opts, - "NnBuilder",NnBuilder.opts, - "NearestNeighborLearner", NearestNeighborLearner.opts, - "Grammar", Grammar.opts, - "Derivation", Derivation.opts, - "Parser", Parser.opts, - "BeamParser", BeamParser.opts, - "SparqlExecutor", SparqlExecutor.opts, - "Dataset", Dataset.opts, - "Params", Params.opts, - "SemanticFn", SemanticFn.opts, - "LexiconFn", LexiconFn.opts, - "Lexicon", Lexicon.opts, - "JoinFn", JoinFn.opts, - "DescriptionValue", DescriptionValue.opts, - "FeatureExtractor", FeatureExtractor.opts, - "LanguageInfo", LanguageInfo.opts, - "EntityLexicon", EntityLexicon.opts, - "FreebaseInfo", FreebaseInfo.opts, - "BridgeFn", BridgeFn.opts, - "WordDistance", WordDistance.opts, - "ContextMapper", ContextModel.opts, - "ParalexRules", ParalexRules.opts, - "ContextModel", ContextModel.opts, - "ContextSimilarityModel", ContextSimilarityModel.opts, - "SubstitutionRuleExtractor", SubstitutionRuleExtractor.opts, - "ParaphraseDataset",ParaphraseDataset.opts, - "ParaphraseLearner", ParaphraseLearner.opts, - "ParaphraseFeatureMatcher", ParaphraseFeatureMatcher.opts, - "Rulebase",Rulebase.opts, - "FormulaRetriever",FormulaRetriever.opts, - "RuleApplier",RuleApplier.opts, - "ParaphraseParser",ParaphraseParser.opts, - "VectorSpaceModel",VectorSpaceModel.opts, - "SyntacticRuleSet",SyntacticRuleSet.opts, - "Aligner",Aligner.opts, - "QuestionGenerator",QuestionGenerator.opts, - "PhraseTable",PhraseTable.opts, - "FeatureSimilarityComputer",FeatureSimilarityComputer.opts, - }); - Execution.run(args, "ParaphraseMain", new ParaphraseMain(), parser); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseParser.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseParser.java deleted file mode 100644 index 4ede010..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseParser.java +++ /dev/null @@ -1,245 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.BooleanValue; -import edu.stanford.nlp.sempre.Evaluation; -import edu.stanford.nlp.sempre.Executor; -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.FormulaGenerationInfo; -import edu.stanford.nlp.sempre.FormulaRetriever; -import edu.stanford.nlp.sempre.Params; - -import fig.basic.Fmt; -import fig.basic.LogInfo; -import fig.basic.Option; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; - -/** - * Paraphrase parsing is done in the following way: - * Given a question |q| - - * 1. Find candidate formulas |(r,e)| relevant for |q| - * 2. Generate candidate questions |q'| for each |(r,e)| - * 3. For every |q'| transfrom q to q' - * 4. extract features from the transformation, r and e and using params score - * 5. return the answer by executing the formula that has the highest score - * @author jonathanberant - * - */ -public class ParaphraseParser { - - public static class Options { - @Option(gloss="Whether to use the vsm model") public boolean vsm=true; - @Option(gloss="Whether to use the alignment model") public boolean alignment=true; - @Option(gloss="Whether to use the alignment model") public boolean baseline=false; - @Option(gloss="Size of beam") public int beamSize=2000; - } - public static Options opts = new Options(); - - private final VectorSpaceModel vsmModel; - private final Aligner aligner; - private final FeatureSimilarityComputer fsComputer; - - private final QuestionGenerator qGenerator; - private final Executor executor; - private final FormulaRetriever fRetriever; - private final ParaphraseFeatureExtractor fExtractor; - - public ParaphraseParser(Executor executor) throws IOException { - qGenerator = new QuestionGenerator(); - fRetriever = new FormulaRetriever(false); - fExtractor = new ParaphraseFeatureExtractor(executor); - vsmModel=VectorSpaceModel.getSingleton(); - aligner=Aligner.getSingleton(); - fsComputer = FeatureSimilarityComputerFactory.getFeatureSimilarityComputer(); - this.executor=executor; - if(opts.baseline && (opts.vsm || - opts.alignment)) - throw new RuntimeException("When using baseline can not use any other model"); - } - - public void parseQuestion(ParsingExample example, Params params) { - - String question = example.utterance; - ensureTargetExecuted(example); - LogInfo.begin_track("Generating candidate formulas"); - List formulaGenerationInfos = fRetriever.retrieveFormulas(example); - LogInfo.logs("Number of candidate formulas=%s",formulaGenerationInfos.size()); - - LogInfo.end_track(); - - LogInfo.begin_track("Generate questions and transform"); - - int numOfGeneratedQuestions=0; - for(FormulaGenerationInfo formulaGenerationInfo: formulaGenerationInfos) { // for every candidate formula - - LogInfo.begin_track("FormulaGenerationInfo=%s",formulaGenerationInfo); - Set generatedQuestions = - qGenerator.getQuestionsForFgInfo(formulaGenerationInfo); //maybe generalize to arbitrary formula - for(String generatedQuestion: generatedQuestions) { //for every generated question - LogInfo.logs("ParaphraseParser.parseQuestion: genQue=%s",generatedQuestion); - ParaphraseExample paraEx = new ParaphraseExample(question, generatedQuestion, new BooleanValue(true)); - - if(opts.vsm) - vsmModel.computeSimilarity(paraEx, params); - if(opts.alignment) - aligner.align(paraEx, params); - if(opts.baseline) - fsComputer.computeSimilarity(paraEx, params); - - - ParaphraseDerivation paraphraseDerivation = - new ParaphraseDerivation(example.languageInfo, - paraEx,formulaGenerationInfo,fExtractor,params); - example.addPrediction(paraphraseDerivation); - numOfGeneratedQuestions++; - } - LogInfo.end_track(); - } - LogInfo.logs("ParaphraseParser.parseQuestions: genQuestions=%s",numOfGeneratedQuestions); - LogInfo.end_track(); - - example.sortDerivations(); - example.predParaDeriv = - new ArrayList<>(example.predParaDeriv.subList(0, Math.min(opts.beamSize, example.predParaDeriv.size()))); - LogInfo.logs("ParphraseParser.parseQuestion: Number of parphrase derivations=%s",example.predParaDeriv.size()); - setEvaluation(example,params); - } - - private void ensureTargetExecuted(ParsingExample example) { - if(example.targetValue==null && example.targetFormula!=null) - example.targetValue = executor.execute(example.targetFormula).value; - } - - /** - * Assumes that praphrasederivations are scored and sorted - */ - private void setEvaluation(ParsingExample ex, Params params) { - final Evaluation eval = new Evaluation(); - int numCandidates = ex.predParaDeriv.size(); - LogInfo.begin_track_printAll("Parser.setEvaluation: %d candidates", numCandidates); - - // Each derivation has a compatibility score (in [0,1]) as well as a model probability. - // Terminology: - // True (correct): compatibility = 1 - // Partial: 0 < compatibility < 1 - // Wrong: compatibility = 0 - - List predictions = ex.predParaDeriv; - - // Make sure derivations are executed - for (ParaphraseDerivation paraphraseDeriv : predictions) { - paraphraseDeriv.ensureExecuted(executor); - } - - // Did we get the answer correct? - int correct_i = -1; // Index of first correct derivation - double maxCompatibility = 0.0; - double[] compatibilities = null; - if (ex.targetValue != null) { - compatibilities = new double[numCandidates]; - for (int i = 0; i < numCandidates; i++) { - ParaphraseDerivation proofDeriv = predictions.get(i); - compatibilities[i] = proofDeriv.compatibility = ex.targetValue.getCompatibility(proofDeriv.value); - - // Must be fully compatible to count as correct. - if (compatibilities[i] == 1 && correct_i == -1) - correct_i = i; - //record maximum compatibility for partial oracle - maxCompatibility = Math.max(compatibilities[i], maxCompatibility); - } - } - - // Compute probabilities - double[] probs = ParaphraseDerivation.getProbs(predictions, 1); - for (int i = 0; i < numCandidates; i++) { - predictions.get(i).prob = probs[i]; - } - -// List> valueList = computeValueList(ex.predParaDeriv); -// evaluateValues(eval, ex, valueList); - - // Number of derivations which have the same top score - int numTop = 0; - double topMass = 0; - if (ex.targetValue != null) { - while (numTop < numCandidates && - compatibilities[numTop] > 0.0d && - Math.abs(predictions.get(numTop).score - predictions.get(0).score) < 1e-10) { - topMass += probs[numTop]; - numTop++; - } - } - double correct = 0; - double partial_correct = 0; - if (ex.targetValue != null) { - for (int i = 0; i < numTop; i++) { - if (compatibilities[i] == 1) correct += probs[i] / topMass; - if (compatibilities[i] > 0) - partial_correct += (compatibilities[i] * probs[i]) / topMass; - } - } - - // Print features (note this is only with respect to the first correct, is NOT the gradient). - // Things are not printed if there is only partial compatability. - if (correct_i != -1 && correct != 1) { - ParaphraseDerivation trueDeriv = predictions.get(correct_i); - ParaphraseDerivation predDeriv = predictions.get(0); - HashMap featureDiff = new HashMap<>(); - trueDeriv.incrementAllFeatureVector(+1, featureDiff); //TODO if features will go out of proof this needs to change - predDeriv.incrementAllFeatureVector(-1, featureDiff); - String heading = String.format("TopTrue (%d) - Pred (%d) = Diff", correct_i, 0); - FeatureVector.logFeatureWeights(heading, featureDiff, params); - } - - // Fully correct - for (int i = 0; i < predictions.size(); i++) { - ParaphraseDerivation deriv = predictions.get(i); - if (compatibilities != null && compatibilities[i] == 1) { - LogInfo.logs( - "True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), - Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); - } - } - // Partially correct - for (int i = 0; i < predictions.size(); i++) { - ParaphraseDerivation deriv = predictions.get(i); - if (compatibilities != null && compatibilities[i] > 0 && compatibilities[i] < 1) { - LogInfo.logs( - "Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), - Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); - } - } - // Anything that's predicted. - for (int i = 0; i < predictions.size(); i++) { - ParaphraseDerivation deriv = predictions.get(i); - // Either print all predictions or this prediction is worse by some amount. - boolean print; - print = probs[i] >= probs[0] / 2 || i < 10; - if (print) { - LogInfo.logs( - "Pred@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(), - Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : ""); - } - } - - eval.add("correct", correct); - eval.add("oracle", correct_i != -1); - eval.add("partCorrect", partial_correct); - eval.add("partOracle", maxCompatibility); - eval.add("numCandidates", numCandidates); // From this parse - if (predictions.size() > 0) - eval.add("parsedNumCandidates", predictions.size()); - - for (ParaphraseDerivation deriv : predictions) { - if (deriv.executorStats != null) - eval.add(deriv.executorStats); - } - // Finally, set all of these stats as the example's evaluation. - ex.setEvaluation(eval); - LogInfo.end_track(); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseUtils.java b/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseUtils.java deleted file mode 100644 index 88d5d84..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParaphraseUtils.java +++ /dev/null @@ -1,179 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.Derivation; -import edu.stanford.nlp.sempre.Example; -import fig.basic.IntPair; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Pair; - -public class ParaphraseUtils { - - public static Map posCanonicMap = new HashMap<>(); - - static { - posCanonicMap.put("JJR", "JJ"); - posCanonicMap.put("JJS", "JJ"); - posCanonicMap.put("NNS", "NN"); - posCanonicMap.put("PRP$", "PRP"); - posCanonicMap.put("RBR", "RB"); - posCanonicMap.put("VBD", "VB"); - posCanonicMap.put("VBG", "VB"); - posCanonicMap.put("VBN", "VB"); - posCanonicMap.put("VBP", "VB"); - posCanonicMap.put("VBZ", "VB"); - posCanonicMap.put("WDT", "W"); - posCanonicMap.put("WP", "W"); - posCanonicMap.put("WP$", "W"); - posCanonicMap.put("WRB", "W"); - } - - public static boolean posCompatible(String antecedentPos, String targetPos) { - String antecedentCanonical = MapUtils.get(posCanonicMap, antecedentPos, antecedentPos); - String targetCanonical = MapUtils.get(posCanonicMap, targetPos, targetPos); - return antecedentCanonical.equals(targetCanonical); } - - public static Pair extractContextIntervalPair(Derivation deriv, Example ex) { - Interval interval = extractEntityInterval(deriv); - String lhs = (0==interval.start) ? "" : ex.lemmaPhrase(0, interval.start); - String rhs = (interval.end == ex.numTokens()) ? "" : ex.lemmaPhrase(interval.end, ex.numTokens()); - Context context = new Context(lhs,rhs); - return Pair.newPair(context, interval); - } - - /** - * This assumes that the derivation contains a single contiguous entity (which is true for entity.grammar) - */ - private static Interval extractEntityInterval(Derivation deriv) { - - Interval res = new Interval(-1, -1); - extractEntityIntervalRecurse(deriv,res); - return res; - } - - private static void extractEntityIntervalRecurse(Derivation deriv, - Interval interval) { - - if(deriv.rule.getLhs() != null && deriv.rule.getLhs().equals("$Entity")) { //base - interval.set(deriv.start, deriv.end); - return; - } - for(Derivation child: deriv.children) { //recurse - extractEntityIntervalRecurse(child,interval); - } - } - - /** - * Computes Levenshtein edit distance between two lists of Strings - stole from Spence that did this on CoreLabel - */ - public static int editDistance(final List l1, final List l2) { - int[][] m = new int[l1.size()+1][l2.size()+1]; - for(int i = 1; i <= l1.size(); i++) - m[i][0] = i; - for(int j = 1; j <= l2.size(); j++) - m[0][j] = j; - - for(int i = 1; i <= l1.size(); i++) { - for(int j = 1; j <= l2.size(); j++) { - m[i][j] = Math.min(m[i-1][j-1] + ((l1.get(i-1).equals(l2.get(j-1))) ? 0 : 1), m[i-1][j] + 1); - m[i][j] = Math.min(m[i][j], m[i][j-1] + 1); - } - } - return m[l1.size()][l2.size()]; - } - - public static boolean isInteger(String str) { - try { - Integer.parseInt(str); - return true; - } - catch(NumberFormatException e) { - return false; - } - } - - /** - * Map with WordNet derivations - */ - public static Map> loadDerivations(String path) { - Map> res = new HashMap<>(); - for(String line: IOUtils.readLines(path)) { - String[] tokens = line.split("\t"); - MapUtils.addToSet(res, tokens[0], tokens[2]); - } - LogInfo.logs("TransUtis.loadDerivations: number of entries=%s",res.size()); - return res; - } - - /** - * Match a sublist in a list - */ - public static boolean matchLists(List list, List pattern) { - for(int i = 0; i < list.size(); ++i) { - if(matchListsFromIndex(list,pattern,i)) - return true; - } - return false; - } - - private static boolean matchListsFromIndex(List list, List pattern, int start) { - - if(start+pattern.size()>list.size()) - return false; - for(int j = 0; j < pattern.size();++j) { - if(!pattern.get(j).equals(list.get(start+j))) - return false; - } - return true; - } - - /** - * Deletes spans that are contained in another span - */ - public static Set getMaxNonOverlappingSpans(Set entitySpans) { - Set res = new HashSet<>(); - for(IntPair pair1: entitySpans) { - boolean containedSpan=false; - for(IntPair pair2: entitySpans) { - if(containedSpan(pair1,pair2)) { - containedSpan=true; - break; - } - } - if(!containedSpan) - res.add(pair1); - } - return res; - } - - /** - * Is pair1 contained in pair2 - */ - public static boolean containedSpan(IntPair pair1, IntPair pair2) { - return (pair1.first >= pair2.first && pair1.second < pair2.second) || - (pair1.first > pair2.first && pair1.second <= pair2.second); - } - - /** - * does span1 intersect with span2 - */ - public static boolean intervalIntersect(IntPair span1, IntPair span2) { - if(span1.first == span1.second || span2.first == span2.second) return false; - if(span1.first>=span2.first && - span1.first= span1.first && - span2.first < span1.second; - - } - - - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParsingExample.java b/src/edu/stanford/nlp/sempre/paraphrase/ParsingExample.java deleted file mode 100644 index 179f36b..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParsingExample.java +++ /dev/null @@ -1,193 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonView; - -import edu.stanford.nlp.sempre.Evaluation; -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.Json; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.Value; -import fig.basic.LogInfo; - -/** - * ParsingExample is an input-out pair. The ParsingExample object stores both the input, - * preprocessing, and output of the parser. - * In ParsingExample we have derivations that are paraphrase mapped to a formula - * @author jonathanberant - * - */ -@JsonIgnoreProperties(ignoreUnknown=true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ParsingExample { - - public static class JsonViews { - public static class WithDerivations {} - public static class WithDPChart {} - } - - //// Information from the input file. - - // Unique identifier for this example. - @JsonProperty public final String id; - - // Input utterance - @JsonProperty public final String utterance; - - // What we should try to predict. - @JsonProperty public Formula targetFormula; // Logical form - @JsonProperty public Value targetValue; // Answer - - //// Information after preprocessing (e.g., tokenization, POS tagging, NER, syntactic parsing, etc.). - @JsonProperty public LanguageInfo languageInfo = null; - - //// Output of the parser. - // Predicted derivations: sorted by score. - @JsonProperty @JsonView(JsonViews.WithDerivations.class) - List predParaDeriv = new ArrayList<>(); - - private Evaluation evaluation; - - public static class Builder { - private String id; - private String utterance; - private Formula targetFormula; - private Value targetValue; - private List predProofDers; - private LanguageInfo languageInfo; - - public Builder setId(String id) { - this.id = id; - return this; - } - public Builder setUtterance(String utterance) { - this.utterance = utterance; - return this; - } - public Builder setTargetFormula(Formula targetFormula) { - this.targetFormula = targetFormula; - return this; - } - public Builder setTargetValue(Value targetValue) { - this.targetValue = targetValue; - return this; - } - public Builder setPredProofs(List predProofDers) { - this.predProofDers = predProofDers; - return this; - } - public Builder setLanguageInfo(LanguageInfo languageInfo) { - this.languageInfo = languageInfo; - return this; - } - public Builder withExample(ParsingExample ex) { - setId(ex.id); - setUtterance(ex.utterance); - setTargetFormula(ex.targetFormula); - setTargetValue(ex.targetValue); - setPredProofs(ex.predParaDeriv); - return this; - } - - public ParsingExample createExample() { - return new ParsingExample( - id, utterance, targetFormula, - targetValue, predProofDers, languageInfo); - } - } - - @JsonCreator - public ParsingExample(@JsonProperty("id") String id, - @JsonProperty("utterance") String utterance, - @JsonProperty("targetFormula") Formula targetFormula, - @JsonProperty("targetValue") Value targetValue, - @JsonProperty("predProofDers") List predProofDers, - @JsonProperty("languageInfo") LanguageInfo languageInfo) { - this.id = id; - this.utterance = utterance; - this.targetFormula = targetFormula; - this.targetValue = targetValue; - this.predParaDeriv = predProofDers == null ? new ArrayList() : predProofDers; - this.languageInfo = languageInfo; - } - - // Accessors - public String getId() { return id; } - public String getUtterance() { return utterance; } - public Evaluation getEvaluation() { return evaluation; } - public int numTokens() { return languageInfo.tokens.size(); } - public List getProofDerivations() { return predParaDeriv; } - public void setTargetFormula(Formula targetFormula) { - this.targetFormula = targetFormula; - } - public void setTargetValue(Value targetValue) { - this.targetValue = targetValue; - } - - public void setEvaluation(Evaluation eval) { evaluation = eval; } - - public String toJson() { - return Json.writeValueAsStringHard(this); - } - - public static ParsingExample fromJson(String json) { - return Json.readValueHard(json, ParsingExample.class); - } - - public void preprocess() { - if (this.languageInfo == null) - this.languageInfo = new LanguageInfo(); - this.languageInfo.analyze(this.utterance); - this.log(); - } - - public List getTokens() { - return languageInfo.tokens; - } - public List getLemmaTokens() { return languageInfo.lemmaTokens; } - - - public void log() { - LogInfo.begin_track("Example: %s", utterance); - LogInfo.logs("Tokens: %s", getTokens()); - LogInfo.logs("Lemmatized tokens: %s", getLemmaTokens()); - LogInfo.logs("POS tags: %s", languageInfo.posTags); - LogInfo.logs("NER tags: %s", languageInfo.nerTags); - LogInfo.logs("NER values: %s", languageInfo.nerValues); - if (targetFormula != null) - LogInfo.logs("targetFormula: %s", targetFormula); - if (targetValue != null) - LogInfo.logs("targetValue: %s", targetValue); - LogInfo.end_track(); - } - - public void clearPredDerivations() { - predParaDeriv.clear(); - } - - public void addPrediction(ParaphraseDerivation p) { - predParaDeriv.add(p); - } - - public void addPredictions(Collection c) { - predParaDeriv.addAll(c); - } - - public void sortDerivations() { - Collections.sort(predParaDeriv, new ParaphraseDerivation.ParaphraseDerivationComparator()); - } - - public void clear() { - for(ParaphraseDerivation paraphraseDerivation: predParaDeriv) - paraphraseDerivation.clear(); - predParaDeriv.clear(); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/ParsingExampleProcessor.java b/src/edu/stanford/nlp/sempre/paraphrase/ParsingExampleProcessor.java deleted file mode 100644 index 47d8a3c..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/ParsingExampleProcessor.java +++ /dev/null @@ -1,120 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.Evaluation; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.Value; -import fig.basic.LogInfo; -import fig.basic.NumUtils; -import fig.basic.Parallelizer; -import fig.basic.StopWatchSet; -import fig.exec.Execution; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ParsingExampleProcessor implements Parallelizer.Processor { - - private final ParaphraseParser paraParser; - private final String prefix; - private final boolean updateWeights; - private Params params; //this is common to threads and should be synchronized - private Evaluation totalEval; //this is common to threads and should be synchronized - private int gcRounds=0; - - public ParsingExampleProcessor(ParaphraseParser paraParser, Params params, String prefix, boolean updateWeights, Evaluation totalEval) { - this.prefix = prefix; - this.paraParser = paraParser; - this.updateWeights=updateWeights; - this.params = params; - this.totalEval = totalEval; - } - - @Override - public void process(ParsingExample ex, int i, int n) { - - LogInfo.begin_track_printAll("%s: example %s/%s:", prefix, i, n); - ex.log(); - Execution.putOutput("example", i); - - StopWatchSet.begin("ParaphraseParser.parseQuestion"); - paraParser.parseQuestion(ex, params); - StopWatchSet.end(); - - if (updateWeights) { - Map counts = computeExpectedCounts(ex, ex.predParaDeriv); - updateWeights(counts); - } - LogInfo.logs("Current: %s", ex.getEvaluation().summary()); - accumulateAndLogEvaluation(ex.getEvaluation(), totalEval, prefix); - ex.clear(); - gcRounds++; - if(gcRounds % 50 ==0) - System.gc(); //due to memory problems - LogInfo.end_track(); - } - - //identical only with ParsingExample and ProofDerivations - private Map computeExpectedCounts(ParsingExample ex, List paraphraseDerivations) { - - Map res = new HashMap<>(); - double[] trueScores; - double[] predScores; - - int n = paraphraseDerivations.size(); - if (n == 0) - return res; - - trueScores = new double[n]; - predScores = new double[n]; - for (int i = 0; i < n; i++) { - ParaphraseDerivation paraphraseDerivation = paraphraseDerivations.get(i); - double r = reward(ex.targetValue, paraphraseDerivation.value); - if (ParaphraseLearner.opts.binaryLogistic) { - trueScores[i] = Math.exp(r); - predScores[i] = 1.0d / (1.0d + Math.exp(-paraphraseDerivation.score)); - } - else { - trueScores[i] = paraphraseDerivation.score + r; - predScores[i] = paraphraseDerivation.score; - } - } - - if(!ParaphraseLearner.opts.binaryLogistic) { - if (!NumUtils.expNormalize(trueScores)) return res; - if (!NumUtils.expNormalize(predScores)) return res; - } - // Update parameters - for (int i = 0; i < n; i++) { - ParaphraseDerivation paraphraseDerivation = paraphraseDerivations.get(i); - double incr = trueScores[i] - predScores[i]; - paraphraseDerivation.incrementAllFeatureVector(incr, res); - } - return res; - } - - private double reward(Value targetValue, Value proofValue) { - if(ParaphraseLearner.opts.partialReward) - return Math.log(targetValue.getCompatibility(proofValue)); - return Math.log(targetValue.getCompatibility(proofValue) == 1 ? 1 : 0); - } - - private void updateWeights(Map counts) { - StopWatchSet.begin("Learner.updateWeights"); - LogInfo.begin_track("Updating weights"); - double sum = 0; - for (double v : counts.values()) sum += v * v; - LogInfo.logs("L2 norm: %s", Math.sqrt(sum)); - params.update(counts); - counts.clear(); - LogInfo.end_track(); - StopWatchSet.end(); - } - - private void accumulateAndLogEvaluation(Evaluation exEvaluation, - Evaluation totalEvaluation, - String prefix) { - totalEvaluation.add(exEvaluation); - LogInfo.logs("Cumulative(%s): %s", prefix, totalEvaluation.summary()); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/Prediction.java b/src/edu/stanford/nlp/sempre/paraphrase/Prediction.java deleted file mode 100644 index f3c99bb..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/Prediction.java +++ /dev/null @@ -1,40 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import edu.stanford.nlp.sempre.Executor; -import edu.stanford.nlp.sempre.Formula; -import edu.stanford.nlp.sempre.Value; - -public class Prediction implements Comparable { - - public final Formula formula; - public final double score; - Value value = null; - double compatibility; - - public Prediction(Formula f, double s) { - this.formula = f; - this.score = s; - } - - public boolean isExecuted() { return value != null; } - - public void ensureExecuted(Executor executor) { - if (!isExecuted()) { - Executor.Response response = executor.execute(formula); - value = response.value; - } - } - - @Override - public int compareTo(Prediction o) { - if(score>o.score) - return -1; - if(score> formulaToLexemsMap = new HashMap>(); - private final String lexiconFile = "lib/fb_data/6/binaryInfoStringAndAlignment.txt"; - - private Map lingAnnotationCache = new HashMap(); - private static final boolean DROP_TYPE1 = true; - private static final Pattern dropRedundantType1Pattern = Pattern.compile("what (.+) is the (.+) of"); - private static final Pattern dropPattern = Pattern.compile("what.* is (.*)"); - - public QuestionGenerator() throws IOException { - Properties props = new Properties(); - props.put("annotators", "tokenize,ssplit,pos,parse"); - pipeline = new StanfordCoreNLP(props); - LogInfo.begin_track("uploading lexicon"); - uploadAlignmentLexicon(); - LogInfo.logs("Number of lexicon formulas: %s",formulaToLexemsMap.size()); - LogInfo.end_track(); - } - - private void uploadAlignmentLexicon() { - for(String line: IOUtils.readLines(lexiconFile)) { - LexiconValue lv = Json.readValueHard(line, LexiconValue.class); - double newCount = MapUtils.getDouble(lv.features, "Intersection_size_typed", 0.0); - if(newCount>opts.alignmentLexiconThreshold) { - if(formulaToLexemsMap.containsKey(lv.formula)) { - double currCount = formulaToLexemsMap.get(lv.formula).getSecond(); - if(newCount>currCount) { - formulaToLexemsMap.put(lv.formula, Pair.newPair(lv.lexeme, newCount)); - } - } - else { - formulaToLexemsMap.put(lv.formula, Pair.newPair(lv.lexeme, newCount)); - } - } - } - } - - /** - * Have explicit entity description - * @param binary - * @param entity - * @return - */ - public Set getQuestionsForFgInfo(FormulaGenerationInfo fgInfo) { - if(opts.bowGeneration) { - return bowGenerate(fgInfo); - } - //generate from formula info - Set res = generateQuestions(fgInfo); - //generate from equivalent formula if it exists - if(fbFormulasInfo.hasOpposite(fgInfo.bInfo.formula)) { - FormulaGenerationInfo eqInfo = - new FormulaGenerationInfo(fbFormulasInfo.getBinaryInfo(fbFormulasInfo.equivalentFormula(fgInfo.bInfo.formula)), - fgInfo.injectedInfo, fgInfo.entityInfo1, fgInfo.entityInfo2, fgInfo.uInfo, fgInfo.isCount, fgInfo.isInject, fgInfo.isUnary); - res.addAll(generateQuestions(eqInfo)); - } - return res; - } - - private Set bowGenerate(FormulaGenerationInfo fgInfo) { - List question = new ArrayList(); - question.add(fgInfo.getQuestionWord()); //question word - String type1Desc = getType1Desc(fgInfo); - if(type1Desc==null) - return Collections.emptySet(); - question.add(type1Desc); - question.add(fgInfo.entityInfo1.desc); - for(String binaryDesc: fgInfo.bInfo.descriptions) - question.add(binaryDesc); - if(fgInfo.isInject) { - question.add(fgInfo.entityInfo2.desc); - } - return Collections.singleton(Joiner.on(' ').join(question)); - } - - /** - * Get questions for a binary formula - main interface with this class - use both FB descriptions and lexicon - * @param bInfo - * @return - */ - private Set generateQuestions(FormulaGenerationInfo fgInfo) { - - Set res = new LinkedHashSet(); - //hack to identify cvt things (I don't trust the CVT markings on freebase schema) - if(fgInfo.bInfo.toReverseString().contains("lambda")) - handleCvtBinary(fgInfo, res); - else - handleNonCvtBinary(fgInfo, res); - - if(opts.genFromLex) { - if(formulaToLexemsMap.containsKey(fgInfo.bInfo.formula)) { - handleLexiconBinary(fgInfo, res); - } - } - res = postProcess(res,fgInfo); - if(fgInfo.isInject) - res = handleInjection(res,fgInfo); - return res; - } - - private Set handleInjection(Set uninjected, FormulaGenerationInfo fgInfo) { - Set injected = new HashSet(); - for(String question: uninjected) { - String injectedQuestion = question.replace("?",Joiner.on(' ').join("in",fgInfo.entityInfo2.desc,"?")); - injected.add(injectedQuestion); - } - return injected; - } - - private Set postProcess(Set questions, FormulaGenerationInfo fgInfo) { - Set res = new LinkedHashSet(); - for(String q: questions) { - String postProcessed = q.replace("what person", "who"); - postProcessed = postProcessed.replace("what date", "when"); - postProcessed=dropRedundantType1(postProcessed); - res.add(postProcessed.trim()); - //possibly add another without type1 - if(DROP_TYPE1 && !fgInfo.isUnary) { - dropType1(postProcessed,res); - } - } - return res; - } - - private static void dropType1(String postProcessed, Set res) { - Matcher m = dropPattern.matcher(postProcessed); - if(m.find()) { - res.add(m.group(1)); - } - } - - private static String dropRedundantType1(String postProcessed) { - Matcher m = dropRedundantType1Pattern.matcher(postProcessed); - if(m.find()) { - String type1 = m.group(1); - String description = m.group(2); - if(type1.startsWith(description) || type1.endsWith(description)) - postProcessed = Joiner.on(' ').join("what is the",type1,"of",postProcessed.substring(m.end()+1)); - else if(description.startsWith(type1) || description.endsWith(type1)) - postProcessed = Joiner.on(' ').join("what is the",description,"of",postProcessed.substring(m.end()+1)); - } - return postProcessed; - } - - private void handleLexiconBinary(FormulaGenerationInfo fgInfo, Set res) { - String binaryDesc = formulaToLexemsMap.get(fgInfo.bInfo.formula).getFirst(); - String type1Desc = getType1Desc(fgInfo); - if(type1Desc==null) - return; - res.add(Joiner.on(' ').join(new String[]{fgInfo.getQuestionWord(),type1Desc,binaryDesc,fgInfo.entityInfo1.desc,"?"})); - } - - private Annotation getAnnotation(String desc) { - synchronized(lingAnnotationCache) { - if(lingAnnotationCache.containsKey(desc)) - return lingAnnotationCache.get(desc); - } - Annotation ann = new Annotation(desc); - synchronized(pipeline) { - pipeline.annotate(ann); - } - synchronized (lingAnnotationCache) { - lingAnnotationCache.put(desc, ann); - } - return ann; - } - - private void handleNonCvtBinary(FormulaGenerationInfo fgInfo, Set res) { - String description = normalizeFbDescription(fgInfo.bInfo.descriptions.get(0)); - Annotation a = getAnnotation(description); - String question = generateNonCvtQuestion(fgInfo, - description, - getPosTagsFromAnnotation(a), - a.get(SentencesAnnotation.class).get(0).get(TreeAnnotation.class).firstChild(), - fbFormulasInfo.isReversed(fgInfo.bInfo.formula)); - if(question!=null) - res.add(question); - } - - private void handleCvtBinary(FormulaGenerationInfo fgInfo, Set res) { - String description1 = normalizeFbDescription(fgInfo.bInfo.descriptions.get(0)); - String description2 = normalizeFbDescription(fgInfo.bInfo.descriptions.get(1)); - Annotation a1 = getAnnotation(description1); - Annotation a2 = getAnnotation(description2); - - String question = generateCvtQuestion(fgInfo,description1,description2, - a1.get(SentencesAnnotation.class).get(0).get(TreeAnnotation.class).firstChild(), - a2.get(SentencesAnnotation.class).get(0).get(TreeAnnotation.class).firstChild(), - getPosTagsFromAnnotation(a1), - getPosTagsFromAnnotation(a2)); - if(question!=null) - res.add(question); - } - - /** - * This method has all the ruls for generating a question - * @param bInfo - * @param binaryDesc - * @param posTags - * @param t - * @param isReversed - * @param isCvt - * @return - */ - private String generateNonCvtQuestion(FormulaGenerationInfo fgInfo, String binaryDesc, - List posTags, Tree t, boolean isReversed) { - - String res; - String type1Desc=getType1Desc(fgInfo); - if(type1Desc==null) //TODO hack - return null; - - String entityDesc = fgInfo.entityInfo1.desc; - String qWord = fgInfo.getQuestionWord(); - binaryDesc=binaryDesc.toLowerCase(); - String category = getTreeCategory(t); - - if(binaryDesc.endsWith("here")) { //special type of description that behaves weirdly - res = handleHere(binaryDesc,isReversed,type1Desc, entityDesc,qWord); - } - else if(category.equals("NP") || category.equals("X") || category.contains("SBARQ") || category.equals("ADVP")) { - res=handleNP(binaryDesc, isReversed, type1Desc, entityDesc, qWord); - } - else if (category.equals("VP") || category.equals("ADJP") || category.equals("SBAR") || category.equals("SINV")) { - res = handleVP(binaryDesc, posTags, t, isReversed, type1Desc, entityDesc,qWord); - } - else if(category.equals("PP")) { - res = handlePP(binaryDesc, isReversed, type1Desc, entityDesc, qWord); - } - else if(category.equals("S")) { - String NP,VP; - if(t.children().length==2 && t.getChild(0).label().toString().equals("NP") && - (t.getChild(1).label().toString().equals("VP") || t.getChild(1).label().toString().equals("ADJP"))) { - NP = yield(t.getChild(0)); - VP = yield(t.getChild(1)); - } - else if(t.children().length==1 && t.getChild(0).label().toString().equals("NP") && - t.getChild(0).getChild(0).label().toString().equals("NP") && - t.getChild(0).numChildren()==2) { - NP = yield(t.getChild(0).getChild(0)); - VP = yield(t.getChild(0).getChild(1)); - } - else if(t.getChild(0).label().toString().equals("VP")) { - NP = ""; - VP = yield(t); - } - else throw new RuntimeException("Unhandled S node: " + t); - res = handleS(binaryDesc, isReversed, type1Desc, entityDesc, NP, VP, qWord); - } - - else if(category.equals("FRAG")) { - if(t.getChild(t.numChildren()-1).label().toString().equals("PP")) { - res = handleFinalPP(fgInfo.bInfo, binaryDesc, isReversed, type1Desc, entityDesc,qWord); - } - else { - if(t.numChildren()==1 && t.getChild(0).label().toString().equals("NP")) { - res = handleNP(binaryDesc, isReversed, type1Desc, entityDesc, qWord); - } - else if(t.numChildren()==1 && t.getChild(0).label().toString().equals("VP")) { - res = handleVP(binaryDesc, posTags, t.getChild(0), isReversed, type1Desc, entityDesc,qWord); - } - else if(t.numChildren()==2 && t.getChild(0).label().toString().equals("NP") && t.getChild(1).label().toString().equals("VP")) { - res = handleS(binaryDesc, isReversed, type1Desc, entityDesc, yield(t.getChild(0)), yield(t.getChild(1)),qWord); - } - else if(posTags.contains("NP")) - res = handleNP(binaryDesc, isReversed, type1Desc, entityDesc, qWord); - else - res = handleVP(binaryDesc, posTags, t, isReversed, type1Desc, entityDesc,qWord); - } - } - else throw new RuntimeException("Not handling " + fgInfo.bInfo+", category="+category); - return res; - } - - private String handlePP(String binaryDesc, boolean isReversed, String type1Desc, String entityDesc, String qWord) { - if(isReversed) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,entityDesc,binaryDesc,"?"}); - else - return Joiner.on(' ').join(new String[]{qWord,type1Desc,binaryDesc,entityDesc,"?"}); - } - - //TODO - maybe simply delete this case - private String handleHere(String description, boolean isReversed, String type1Desc, String entityDesc, String qWord) { - String[] tokens = description.split("\\s+"); - if(isReversed) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",tokens[tokens.length-2],"in",entityDesc,"?"}); - else - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",entityDesc,tokens[tokens.length-2],"in","?"}); - } - - private String handleFinalPP(BinaryFormulaInfo bInfo, String description, - boolean isReversed, String type1Desc, String entityDesc, String qWord) { - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",description,entityDesc,"?"}); - } - - private String handleVP(String description, List posTags, Tree t, - boolean isReversed, String type1Desc, String entityDesc, String qWord) { - description=dropTrailingNP(description,t); - if(isReversed) { - if(posTags.contains("VBN")) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",entityDesc,description,"?"}); - else if(posTags.contains("VBD")) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"did",entityDesc,description,"?"}); - else if(posTags.contains("VBZ")) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"does",entityDesc,description,"?"}); - else - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"do",entityDesc,description,"?"}); - } - else { - if(posTags.contains("VBN")) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",description,entityDesc,"?"}); - else - return Joiner.on(' ').join(new String[]{qWord,type1Desc,description,entityDesc,"?"}); - } - } - - private String handleS(String description, boolean isReversed, - String type1Desc, String entity, String NP, String VP, String qWord) { - if(isReversed) { - if(description.endsWith("by")) { //TODO try getting rid of this case - VP = VP.substring(0,VP.indexOf("by")); - return Joiner.on(' ').join(new String[]{qWord,type1Desc,VP,entity,"?"}); - } - else - return Joiner.on(' ').join(new String[]{qWord,NP,"is",VP,"by",entity,"?"}); - } - else { - if(description.endsWith("by")) //TODO try getting rid of this case - return Joiner.on(' ').join(new String[]{qWord,NP,"is",VP,entity,"?"}); - else - return Joiner.on(' ').join(new String[]{qWord,type1Desc,VP,"the",NP,entity,"?"}); - } - } - - private String handleNP(String description, boolean isReversed, - String type1Desc, String entityDesc, String qWord) { - if(isReversed) { - if(description.endsWith("of")) - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is","the",description,entityDesc,"?"}); - else - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"is","the",description,"of",entityDesc,"?"}); - } - else { - return Joiner.on(' ').join(new String[]{qWord,type1Desc,"has",entityDesc,"as",description,"?"}); - } - } - - private String generateCvtQuestion(FormulaGenerationInfo fgInfo, - String description1, String description2, Tree t1, Tree t2, - List posTags1, List posTags2) { - - boolean isReversed = fbFormulasInfo.isReversed(fgInfo.bInfo.formula); - String type1Desc = getType1Desc(fgInfo); - if(type1Desc==null) - return null; - String qWord = fgInfo.getQuestionWord(); - String entityDesc = fgInfo.entityInfo1.desc; - - description1=description1.toLowerCase(); - description2=description2.toLowerCase(); - String root1 = t1.label().toString(); - String root2 = t2.label().toString(); - String res; - if(!isReversed) { - //NP NP - if( (isNPorXorSorFRAG(root1) || root1.equals("PP") || root1.equals("SINV") || root1.equals("SBAR")) - && - isNPorXorSorFRAG(root2)) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"has the",description2,entityDesc,"?"}); - //VP NP - else if( (isADJPorVPorUCP(root1) && isNPorXorSorFRAG(root2))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,description1,"has the",description2,entityDesc,"?"});//VP NP non, ADJP NP non - //NP VP or VP VP - else if(isNPorXorSorFRAG(root1) && isADJPorVPorUCP(root2) || - isADJPorVPorUCP(root1) && isADJPorVPorUCP(root2)) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",description2,entityDesc,"?"}); //NP VP non, NP ADJP non, S NP non - //other - else if((root1.equals("VP") && root2.equals("SINV")) || - (root1.equals("NP") && root2.equals("ADVP")) || - (root1.equals("NP") && root2.equals("PP")) || - (root1.equals("NP") && root2.equals("SINV"))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,description1,description2,entityDesc,"?"}); //NP FRAG non - else - throw new RuntimeException("Does not handle: "+fgInfo.bInfo+", root1=" + root1 + ", root2="+root2+", is reversed="+isReversed); - } - else { - if(isNPorFRAG(root1) && isNPorXorFRAG(root2)) { - if(FreebaseInfo.isPrimitive(fgInfo.bInfo.expectedType1)) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is the",description1,"of",description2,"of",entityDesc,"?"}); //NP NP reverse, NP FRAG reverse, FRAG S reverse - else - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is the",description1,"of",entityDesc,"?"}); //NP NP reverse, NP FRAG reverse, FRAG S reverse - } - - else if(isNPorXorFRAG(root1) && root2.equals("S")) //the S gives verb that provides information that's why we use description2 and not description1 - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is the",description2,"by",entityDesc,"?"});//NP S reverse - - else if(root1.equals("S") && root2.equals("S") || - root1.equals("S") && root2.equals("FRAG")) //maybe should split if its N V or V N - think a bit more - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is the",description1,"by",entityDesc,"?"}); //S S reverse, - - else if(root1.equals("S") && root2.equals("NP")) //maybe should think about this one a bit more - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",entityDesc,"a",description2,"of"});//S NP reverse - - else if((isADJPorVPorUCP(root1) && isNPorXorSorFRAG(root2)) || - (root1.equals("SINV") && isNPorXorFRAG(root2))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",entityDesc,description1}); //VP NP reverse, ADJP NP rev, VP S reverse - - //added for a case where there were pos errors so this is just so that there is no exception - else if((root1.equals("VP") && root2.equals("VP"))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"do",entityDesc,description1}); //VP NP reverse, ADJP NP rev, VP S reverse - - else if(isNPorXorSorFRAG(root1) && isADJPorVPorUCP(root2) || - isNPorXorSorFRAG(root1) && root2.equals("PP")) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",description1,"that",entityDesc,description2,"?"}); //NP VP rev, NP ADJP rev - - else if(root1.equals("SINV") && root2.equals("VP")) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,description1,"that",entityDesc,description2,"?"}); //NP VP rev, NP ADJP rev - - else if((isNPorXorSorFRAG(root1) || root1.equals("ADJP")) - && root2.equals("SINV")) - res = Joiner.on(' ').join(new String[]{qWord,"is the",type1Desc,"for",entityDesc,"that",description2}); //NP SINV rev - - else if((root1.equals("NP") && root2.equals("SBAR"))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is the",description2,"of",entityDesc,"?"}); - //handle things that start with PP - else if((root1.equals("ADVP") || root1.equals("PP") || root1.equals("X")) - && (root2.equals("NP") || root2.equals("FRAG") || root2.equals("ADJP") || root2.equals("X"))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is the",description1,"in",description2,"of",entityDesc,"?"}); - - else if(((root1.equals("ADVP") || root1.equals("PP")) - && root2.equals("S"))) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",description1,"of",description2,"by",entityDesc,"?"}); //NP VP rev, NP ADJP rev - - else if(root1.equals("PP") - && root2.equals("VP")) - res = Joiner.on(' ').join(new String[]{qWord,type1Desc,"is",description1,"of",entityDesc,description2,"?"}); //NP VP rev, NP ADJP rev - - else - throw new RuntimeException("Does not handle: "+fgInfo.bInfo+", root1=" + root1 + ", root2="+root2+", is reversed="+isReversed); - } - if(opts.verbose>=3) - LogInfo.logs("QuestionGenration: binary=%s, expType1=%s, exType2=%s, description1=%s, description2=%s, root1=%s, root2=%s, isReverse=%s, res=%s", - fgInfo.bInfo.formula,fgInfo.bInfo.expectedType1,fgInfo.bInfo.expectedType2,description1,description2,root1,root2,isReversed,res); - return res; - } - - private String getType1Desc(FormulaGenerationInfo fgInfo) { - if(fgInfo.isUnary) { - return fgInfo.uInfo.getRepresentativeDescrption(); - } - if(fgInfo.bInfo.expectedType1.equals(FreebaseInfo.DATE)) - return "date"; - if(fgInfo.bInfo.expectedType1.equals(FreebaseInfo.FLOAT)) - return "number"; - if(fgInfo.bInfo.expectedType1.equals(FreebaseInfo.INT)) - return "number"; - if(fgInfo.bInfo.expectedType1.equals(FreebaseInfo.BOOLEAN)) - return ""; - if(fgInfo.bInfo.expectedType1.equals(FreebaseInfo.TEXT)) - return "description"; - Formula type1Formula = new JoinFormula(FreebaseInfo.TYPE, new ValueFormula(new NameValue(fgInfo.bInfo.expectedType1))); - try { - if(fbFormulasInfo.getUnaryInfo(type1Formula)==null) { - LogInfo.logs("No unary info for=%s",type1Formula); - return null; - } - return fbFormulasInfo.getUnaryInfo(type1Formula).getRepresentativeDescrption(); - } - catch(NullPointerException e) { - if(type1Formula.toString().equals("(fb:type.object.type fb:type.object)")) - return "thing"; - else { - LogInfo.logs("Binfo exType1=%s, exType2=%s",fgInfo.bInfo.expectedType1,fgInfo.bInfo.expectedType2); - throw new RuntimeException(e); - } - } - } - - //STATIC METHODS - private static String dropTrailingNP(String description, Tree t) { - - if(t.numChildren()==2 && - t.getChild(0).label().toString().startsWith("VB") && - t.getChild(1).label().toString().equals("PP") && - t.getChild(1).numChildren()==2 && - t.getChild(1).getChild(0).label().toString().equals("IN") && - t.getChild(1).getChild(1).label().toString().equals("NP")) { - description = yield(t.getChild(0)) + " " + yield(t.getChild(1).getChild(0)); - - } - return description; - } - - private static String getTreeCategory(Tree t) { - String rootLabel = t.label().toString(); - if (rootLabel.equals("S") && t.numChildren()==1 && t.getChild(0).label().toString().equals("VP") - && t.getChild(0).getChild(0).label().toString().equals("VBG")) - return "NP"; - if(rootLabel.equals("S") && t.numChildren()==1 && t.getChild(0).label().toString().equals("VP") - && t.getChild(0).getChild(0).label().toString().equals("VBN")) - return "VP"; - return rootLabel; - } - - private static List getPosTagsFromAnnotation(Annotation a) { - List posTags = new ArrayList(); - for(CoreLabel token: a.get(TokensAnnotation.class)) - posTags.add(token.get(PartOfSpeechAnnotation.class)); - return posTags; - } - - private static String yield(Tree t) { - StringBuilder sb = new StringBuilder(); - for(Word word: t.yieldWords()) - sb.append(word.word()+" "); - return sb.toString().trim(); - } - - private static String normalizeFbDescription(String description) { - description=description.replace("this", "the"); - int startBracket = description.indexOf('('); - int endBracket = description.indexOf(')'); - if(startBracket!=-1 && endBracket!=-1) - description=description.substring(0,startBracket)+description.substring(endBracket+1); - return description; - } - - - private static boolean isNPorXorSorFRAG(String str) {return str.equals("NP") || str.equals("X") || str.equals("S") || str.equals("FRAG"); } - private static boolean isNPorXorFRAG(String str) {return str.equals("NP") || str.equals("X") || str.equals("FRAG"); } - private static boolean isNPorFRAG(String str) {return str.equals("NP") || str.equals("FRAG"); } - private static boolean isADJPorVPorUCP(String str) {return str.equals("VP") || str.equals("ADJP") || str.equals("UCP"); } - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/VectorSpaceModel.java b/src/edu/stanford/nlp/sempre/paraphrase/VectorSpaceModel.java deleted file mode 100644 index d7b34c4..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/VectorSpaceModel.java +++ /dev/null @@ -1,288 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import com.google.common.base.Strings; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.BooleanValue; -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.LanguageInfo.LanguageUtils; -import fig.basic.ListUtils; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.NumUtils; -import fig.basic.Option; -import fig.basic.MemUsage; -import fig.basic.Pair; - -/** - * Provides similarity score for a pair of phrases |x| and |x'| by representing them - * a pair of vectors and computing similarity with some parameters - e.g. - * s(x,x')=xWx' where W is a matrix (diagonal or not...) etc. - * @author jonathanberant - */ -public class VectorSpaceModel implements MemUsage.Instrumented,FeatureSimilarityComputer { - - public enum PhraseRep { - ADDITIVE, - CW_ADDITIVE, - AVG, - CW_AVG; - - public static PhraseRep parse(String str) { - if("additive".equals(str)) - return ADDITIVE; - if("cw_additive".equals(str)) - return CW_ADDITIVE; - if("avg".equals(str)) - return AVG; - if("cw_avg".equals(str)) - return CW_AVG; - throw new RuntimeException("Illegal mode: " + str); - } - } - - public enum SimilarityFunc { - DIAGNONAL, - DOT_PROD, - FULL_MATRIX; - - public static SimilarityFunc parse(String str) { - if("diagonal".equals(str)) - return DIAGNONAL; - if("full_matrix".equals(str)) - return FULL_MATRIX; - if("dot_product".equals(str)) - return DOT_PROD; - throw new RuntimeException("Illegal mode: " + str); - } - } - - public static class Options { - @Option(gloss = "Path to file containing word vectors, one per line") public String wordVectorFile; - @Option(gloss = "Vector dimension") public int vecCapacity=50; - @Option(gloss = "VSM model phrase representation") public String phraseRep="cw_avg"; - @Option(gloss = "VSM model similarity function") public String similarityFunc="full_matrix"; - @Option(gloss = "verbose") public int verbose = 0; - } - public static Options opts = new Options(); - private static VectorSpaceModel vsm; - - private Map wordVectors; - private static Map phraseVectorCache; //not clear this is necessary for efficiency - private PhraseRep vsmPhraseRep; - private SimilarityFunc vsmSimilarityFunc; - - public static VectorSpaceModel getSingleton() { - if(vsm==null) - vsm = new VectorSpaceModel(); - return vsm; - } - - private VectorSpaceModel() { - vsmPhraseRep=PhraseRep.parse(opts.phraseRep); - vsmSimilarityFunc=SimilarityFunc.parse(opts.similarityFunc); - wordVectors = new HashMap(); - phraseVectorCache = new HashMap(); - if (Strings.isNullOrEmpty(opts.wordVectorFile)) - return; - - String header = null; - for (String line : IOUtils.readLines(opts.wordVectorFile)) { - String[] tokens = line.split("\\s+"); - // Some word embedding files have a header which includes the number of - // words and the number of dimensions. Ignore this. - if (header == null && tokens.length == 2) { - header = line; - continue; - } - if (tokens.length - 1 != opts.vecCapacity) - throw new RuntimeException("Expected " + opts.vecCapacity + " tokens, but got " + (tokens.length-1) + ": " + line); - double[] vector = new double[opts.vecCapacity]; - for (int i = 1; i < tokens.length; ++i) - vector[i-1]=Double.parseDouble(tokens[i]); - wordVectors.put(tokens[0], vector); - } - } - - public void computeSimilarity(ParaphraseExample ex, Params params) { - - ex.ensureAnnotated(); - //get source and target representations - double[] sourceVec,targetVec; - synchronized (phraseVectorCache) { - sourceVec = phraseVectorCache.containsKey(ex.source) ? phraseVectorCache.get(ex.source) : computeUtteranceVec(ex.sourceInfo); - targetVec = phraseVectorCache.containsKey(ex.target) ? phraseVectorCache.get(ex.target) : computeUtteranceVec(ex.targetInfo); - MapUtils.putIfAbsent(phraseVectorCache, ex.source, sourceVec); - MapUtils.putIfAbsent(phraseVectorCache, ex.target, targetVec); - } - //combine them - FeatureVector fv; - if(vsmSimilarityFunc==SimilarityFunc.DIAGNONAL) - fv = getDiagonalMatrixFeatures(sourceVec,targetVec); - else if(vsmSimilarityFunc==SimilarityFunc.FULL_MATRIX) - fv = getFullMatrixFeatures(sourceVec,targetVec); - else //dot product - fv = getDotProductFeature(sourceVec,targetVec); - //set stuff - ex.setVectorSpaceSimilarity(new FeatureSimilarity(fv,ex.source,ex.target,params)); - } - - private FeatureVector getDotProductFeature(double[] sourceVec, double[] targetVec) { - FeatureVector res = new FeatureVector(); - res.add("VS","dot_product",ListUtils.dot(sourceVec, targetVec)); - return res; - } - - private FeatureVector getFullMatrixFeatures(double[] sourceVec, - double[] targetVec) { - //FeatureVector res = new FeatureVector(); - FeatureVector res = new FeatureVector(opts.vecCapacity*opts.vecCapacity); - int featureNum=0; - for(int i = 0; i < sourceVec.length; ++i) { - for(int j = 0; j < targetVec.length; j++) { - res.addDenseFeature(featureNum++, sourceVec[i]*targetVec[j]); - //res.add("VS", "d"+i+",d"+j, sourceVec[i]*targetVec[j]); - } - } - return res; - } - - private FeatureVector getDiagonalMatrixFeatures(double[] source, - double[] target) { - FeatureVector res = new FeatureVector(opts.vecCapacity); - int featureNum=0; - for(int i = 0; i < source.length; ++i) { - res.addDenseFeature(featureNum++, source[i]*target[i]); - } - return res; - } - - /** - * The vec representation of a phrase is the some of its words - * @param langInfo - * @return - */ - private double[] computeUtteranceVec(LanguageInfo langInfo) { - - double[] res = new double[opts.vecCapacity]; - int numOfAddedTokens=0; - for(int i = 0; i < langInfo.numTokens(); ++i) { - String pos = langInfo.posTags.get(i); - if((vsmPhraseRep==PhraseRep.CW_ADDITIVE || vsmPhraseRep==PhraseRep.CW_AVG) - && !LanguageUtils.isContentWord(pos)) - continue; - - double[] tokenVec = wordVectors.get(langInfo.tokens.get(i)); - if(tokenVec!=null) { - ListUtils.addMut(res, tokenVec); - numOfAddedTokens++; - } - } - if((vsmPhraseRep==PhraseRep.AVG || vsmPhraseRep==PhraseRep.CW_AVG) - && numOfAddedTokens > 0) { - double inverse = (double) 1 / numOfAddedTokens; - res = ListUtils.mult(inverse, res); - } - return res; - } - - /** - * Holds the similarity features (e.g. component wise product or outer product etc.) - * @author jonathanberant - * - */ - - @Override - public long getBytes() { - return MemUsage.objectSize(MemUsage.pointerSize*2)+ - MemUsage.getBytes(wordVectors)+MemUsage.getBytes(phraseVectorCache); - } - - public static long cacheSize() { - return MemUsage.getBytes(phraseVectorCache); - } - - private void printWordSimilarity(ParaphraseExample paraExample, Params params) { - double alpha = 0.02; - List> scoreList = new LinkedList>(); - paraExample.ensureAnnotated(); - for(int i = 0; i < paraExample.sourceInfo.numTokens(); ++i) { - String sourcePos = paraExample.sourceInfo.posTags.get(i); - if((vsmPhraseRep==PhraseRep.CW_ADDITIVE || vsmPhraseRep==PhraseRep.CW_AVG) - && !LanguageUtils.isContentWord(sourcePos)) - continue; - - double[] sourceTokenVec = wordVectors.get(paraExample.sourceInfo.tokens.get(i)); - if(sourceTokenVec!=null) { - for(int j = 0; j < paraExample.targetInfo.numTokens(); ++j) { - String targetPos = paraExample.targetInfo.posTags.get(j); - if((vsmPhraseRep==PhraseRep.CW_ADDITIVE || vsmPhraseRep==PhraseRep.CW_AVG) - && !LanguageUtils.isContentWord(targetPos)) - continue; - String token1 = paraExample.sourceInfo.tokens.get(i); - String token2 = paraExample.targetInfo.tokens.get(j); - if((token1.equals("czech") || token1.equals("republic")) && - (token2.equals("czech") || token2.equals("republic"))) - continue; - - double[] targetTokenVec = wordVectors.get(paraExample.targetInfo.tokens.get(j)); - if(targetTokenVec!=null) { - FeatureVector fv; - if(vsmSimilarityFunc==SimilarityFunc.FULL_MATRIX) - fv = getFullMatrixFeatures(sourceTokenVec, targetTokenVec); - else if(vsmSimilarityFunc==SimilarityFunc.DIAGNONAL) - fv = getDiagonalMatrixFeatures(sourceTokenVec, targetTokenVec); - else - fv = getDotProductFeature(sourceTokenVec, targetTokenVec); - double score = fv.dotProduct(params); - scoreList.add(Pair.newPair(paraExample.sourceInfo.tokens.get(i)+","+paraExample.targetInfo.tokens.get(j), alpha*score)); - } - } - } - } - double[] scores = new double[scoreList.size()]; - String[] tokens = new String[scoreList.size()]; - for(int i = 0; i < scoreList.size();++i) { - tokens[i] = scoreList.get(i).getFirst(); - scores[i] = scoreList.get(i).getSecond(); - } - NumUtils.expNormalize(scores); - for(int i = 0; i < scores.length; ++i) - LogInfo.log(tokens[i]+"\t"+scores[i]); - } - - public static void main(String[] args) throws IOException { - opts.wordVectorFile = "/Users/jonathanberant/Projects/semparse/lib/wordreprs/cbow-lowercase-50.vectors"; - ParaphraseExample paraExample =new ParaphraseExample("what do people in the czech republic speak?", - "the official language of czech republic ?",new BooleanValue(true)); - if(args[0].equals("full_matrix")) { - opts.similarityFunc="full_matrix"; - VectorSpaceModel vsm = VectorSpaceModel.getSingleton(); - Params params = new Params(); - params.read("/Users/jonathanberant/Research/temp/918params"); //full matrix - vsm.printWordSimilarity(paraExample,params); - } - else if(args[0].equals("diagonal")) { - opts.similarityFunc="diagonal"; - VectorSpaceModel vsm = VectorSpaceModel.getSingleton(); - Params params = new Params(); - params.read("/Users/jonathanberant/Research/temp/949params"); //diagonal - vsm.printWordSimilarity(paraExample,params); - } - else { - opts.similarityFunc="dot_product"; - VectorSpaceModel vsm = VectorSpaceModel.getSingleton(); - Params params = new Params(); - params.read("/Users/jonathanberant/Research/temp/954params"); //diagonal - vsm.printWordSimilarity(paraExample,params); - } - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexQuestionReader.java b/src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexQuestionReader.java deleted file mode 100644 index 41db4e1..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexQuestionReader.java +++ /dev/null @@ -1,47 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.paralex; - -import java.io.BufferedReader; -import java.io.IOException; - -import edu.stanford.nlp.io.IOUtils; - -/** - * Takes the question file from paralex and reads questions - * @author jonathanberant - * - */ -public class ParalexQuestionReader { - - private String questionFile = "/u/nlp/data/semparse/paralex/wikianswers-paraphrases-1.0/questions.txt"; - private BufferedReader reader; - - public ParalexQuestionReader() throws IOException { - reader = IOUtils.getBufferedFileReader(questionFile); - } - - public String next() throws IOException { - - String res = null; - boolean cont = true; - while(cont) { - String line = reader.readLine(); - if(line==null) - cont=false; - else { - String[] tokens = line.split("\t"); - if(tokens.length==4) { - cont=false; - res = tokens[0]; - } - } - } - return res; - } - - public void close() throws IOException { - reader.close(); - } - - - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexRules.java b/src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexRules.java deleted file mode 100644 index c97531b..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexRules.java +++ /dev/null @@ -1,190 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.paralex; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.LanguageInfo.LanguageUtils; -import edu.stanford.nlp.sempre.fbalignment.utils.DoubleContainer; -import edu.stanford.nlp.sempre.paraphrase.Context; -import edu.stanford.nlp.sempre.paraphrase.Interval; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Option; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ParalexRules { - - public static class Options { - @Option public int verbose = 0; - @Option public String paralexQuestions; - @Option public String paralexAlignments; - } - public static Options opts = new Options(); - - private Map lemmaToInterval = new HashMap(); - private Map> rulesMap = new HashMap>(); - - public ParalexRules() { - LogInfo.begin_track_printAll("Building rule map"); - populateLemmaToPosMap(); - extractRules(); - lemmaToInterval.clear(); //to save memory - if(opts.verbose>3) - log(); - LogInfo.end_track(); - } - - private void log() { - for(Context c1: rulesMap.keySet()) { - for(Context c2: rulesMap.get(c1).keySet()) { - LogInfo.log("ParalexRules.log:\t"+c1+"\t"+c2+"\t"+rulesMap.get(c1).get(c2).value()); - } - } - } - - private void extractRules() { - - int numOfNoNounRules = 0; - int numOfBadAlignments = 0; - int numOfUsedParaphrases = 0; - for(String line: IOUtils.readLines(opts.paralexAlignments)) { - String[] tokens = line.split("\t"); - if(!lemmaToInterval.containsKey(tokens[0]) || !lemmaToInterval.containsKey(tokens[1])) { - numOfNoNounRules++; - continue; - } - Interval interval1 = lemmaToInterval.get(tokens[0]); - Interval interval2 = lemmaToInterval.get(tokens[1]); - if(nounNounAlignment(tokens[2],interval1,interval2)) { - List question1 = Arrays.asList(tokens[0].split("\\s+")); - List question2 = Arrays.asList(tokens[1].split("\\s+")); - Context context1 = new Context(question1, interval1); - Context context2 = new Context(question2, interval2); - if(!context1.equals(context2)) { - insertRule(context1,context2); - insertRule(context2,context1); - } - numOfUsedParaphrases++; - } - else { - numOfBadAlignments++; - } - } - LogInfo.logs("ParalexRules.extractRules: number of rules without noun: %s, number of bad alignments=%s, number of used paraphrases=%s",numOfNoNounRules, - numOfBadAlignments,numOfUsedParaphrases); - } - - private void insertRule(Context context1, Context context2) { - Map context1Rules = rulesMap.get(context1); - if(context1Rules==null) { - context1Rules = new HashMap(); - rulesMap.put(context1, context1Rules); - } - DoubleContainer count = context1Rules.get(context2); - if(count==null) { - count = new DoubleContainer(0.0); - context1Rules.put(context2, count); - } - count.set(count.value()+1); - } - - private boolean nounNounAlignment(String alignmentDesc, Interval interval1, - Interval interval2) { - - String[] alignments = alignmentDesc.split("\\s+"); - for(int i = 0; i < alignments.length; ++i) { - String[] pair = alignments[i].split("-"); - Integer index1 = Integer.parseInt(pair[0]); - Integer index2 = Integer.parseInt(pair[1]); - if((interval1.contains(index1) && !interval2.contains(index2)) || - (!interval1.contains(index1) && interval2.contains(index2))) { - return false; - } - } - return true; - } - - private void populateLemmaToPosMap() { - - int numOfLinesWithLessThanFourTokens = 0; - int numOfLinesWithDifferentNumberOfTokens = 0; - int numOfLinesWithNotOneNoun = 0; - for(String line: IOUtils.readLines(opts.paralexQuestions)) { - String[] tokens = line.split("\t"); - if(tokens.length<4) { - numOfLinesWithLessThanFourTokens++; - continue; - } - - String[] posTags = tokens[2].split("\\s+"); - String[] lemmas = tokens[3].split("\\s+"); - if(posTags.length!=lemmas.length) { - numOfLinesWithDifferentNumberOfTokens++; - continue; - } - Interval interval = getNounInterval(posTags); - if(interval!=null) { - if(opts.verbose>=3) - LogInfo.logs("ParalexRules.populateLemmaToPos: a single noun=%s",line); - lemmaToInterval.put(tokens[3], interval); - } - else { - if(opts.verbose>=3) - LogInfo.logs("ParalexRules.populateLemmaToPos: not a single noun=%s",line); - numOfLinesWithNotOneNoun++; - } - } - LogInfo.logs("lines with less than four fields: %s, lines with different num of pos and lemmas: %s, lines with not 1 NN: %s, " + - "lines uploaded=%s", - numOfLinesWithLessThanFourTokens, - numOfLinesWithDifferentNumberOfTokens,numOfLinesWithNotOneNoun,lemmaToInterval.size()); - } - - /** - * get noun interval, if there is more than one noun interval or none - return null - * @param posTags - * @return - */ - private Interval getNounInterval(String[] posTags) { - int start=-1,end=-1; - //find first noun - for(int i = 0; i < posTags.length; ++i) { - if(LanguageUtils.isProperNoun(posTags[i])) { - start=i; - break; - } - } - //find last noun - for(int i = 0; i < posTags.length; ++i) { - if(LanguageUtils.isProperNoun(posTags[posTags.length-1-i])) { - end=posTags.length-i; - break; - } - } - //if no nouns return null - if(end<=start) - return null; - //if more than one noun return null - for(int i = start+1; i < end; ++i) { - if(!LanguageUtils.sameProperNounClass(posTags[i], posTags[i-1])) - return null; - } - return new Interval(start, end); - } - - public DoubleContainer match(Context questionContext, Context candidate) { - if(rulesMap.containsKey(questionContext)){ - return MapUtils.get(rulesMap.get(questionContext),candidate,new DoubleContainer(0.0)); - } - return new DoubleContainer(0.0); - } - - public static void main(String[] args) { - opts.paralexQuestions = args[0]; - opts.paralexAlignments = args[1]; - ParalexRules rules = new ParalexRules(); - rules.log(); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/paralex/PhraseTable.java b/src/edu/stanford/nlp/sempre/paraphrase/paralex/PhraseTable.java deleted file mode 100644 index 0d23519..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/paralex/PhraseTable.java +++ /dev/null @@ -1,63 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.paralex; - -import java.util.HashMap; -import java.util.Map; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.paraphrase.Aligner.AlignmentStats; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Option; - -public class PhraseTable { - - public static class Options { - @Option(gloss="Path to file with phrase table") public String phraseTablePath="lib/paralex/phrase-table.counts.txt"; - @Option(gloss="Minimum phrase table count") public int phraseTableThresh=5; - } - public static Options opts = new Options(); - - public static PhraseTable phraseTable; - - Map> table; - - public static PhraseTable getSingleton() { - if(phraseTable==null) { - phraseTable = new PhraseTable(); - } - return phraseTable; - } - - private PhraseTable() { - table = loadPhraseTable(opts.phraseTablePath, opts.phraseTableThresh); - } - - /** - * Loading phrase table from file - * @return - */ - public Map> loadPhraseTable(String path, int threshold) { - LogInfo.begin_track("Loading phrase table"); - Map> res = new HashMap>(); - for(String line: IOUtils.readLines(path)) { - String[] tokens = line.split("\t"); - double count = Double.parseDouble(tokens[2]); - MapUtils.putIfAbsent(res, tokens[0], new HashMap()); - if(count>=threshold) { - AlignmentStats aStats = new AlignmentStats(count, Double.parseDouble(tokens[3]), Double.parseDouble(tokens[4])); - res.get(tokens[0]).put(tokens[1], aStats); - } - } - LogInfo.logs("ParaphraseUtils.loadPhraseTable: number of entries=%s",res.size()); - LogInfo.end_track(); - return res; - } - - public boolean containsKey(String lhsPhrase) { - return table.containsKey(lhsPhrase); - } - - public Map get(String lhsPhrase) { - return table.get(lhsPhrase); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/LangItem.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/LangItem.java deleted file mode 100644 index 007d5e7..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/LangItem.java +++ /dev/null @@ -1,191 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.lang.reflect.Array; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.paraphrase.Interval; -import edu.stanford.nlp.sempre.paraphrase.rules.LanguageExpToken.RepeatType; -import edu.stanford.nlp.sempre.paraphrase.rules.LanguageExpToken.TokenType; -import fig.basic.LispTree; -import fig.basic.MapUtils; - -/** - * A regexp that matches to a LanguageInfo - * The format of an item is (item (pos [NN]+) (ner [DATE]*)) - * @author jonathanberant - * - */ -public class LangItem { - - public final LispTree expressionTree; - private NFA nfa; - - public LangItem(LispTree tree) { - expressionTree = tree; - nfa = new NFA(expressionTree); - } - - public LispTree toLispTree() { - return expressionTree; - } - - public String toString() { - return expressionTree.toString(); - } - - public boolean match(LanguageInfo utterance, int start, int end) { - return match(utterance, start).contains(new Interval(start, end)); - } - - public List match(LanguageInfo utterance, int start) { - - List res = new ArrayList(); - res.addAll(nfa.match(utterance, start)); - return res; - } - - /** - * Language item is defined to contain just a single language expression token - * @return - */ - public LanguageExpToken getLangExpToken() { - return new LanguageExpToken(expressionTree.child(1).child(0).value, - expressionTree.child(1).child(1).value); - } - - public static class NFA { - - private int numOfStates; - private Map>[] outgoingEdges; - private Map>[] incomingEdges; - private Set startStates = new LinkedHashSet(); - - @SuppressWarnings("unchecked") - public NFA(LispTree expressionTree) { - numOfStates=1; - startStates.add(0); - outgoingEdges = (Map>[]) - Array.newInstance(HashMap.class,expressionTree.children.size()); - incomingEdges = (Map>[]) - Array.newInstance(HashMap.class,expressionTree.children.size()); - compileExpression(expressionTree); - } - - private void compileExpression(LispTree exp) { - - for(int i = 1; i < exp.children.size();++i) { - - LispTree child = exp.child(i); - int currState = numOfStates-1; - LanguageExpToken langExpToken = new LanguageExpToken(child.child(0).value, child.child(1).value); - - if(langExpToken.repeat==RepeatType.PLUS) { - addEdge(currState, currState+1,langExpToken); - addEdge(currState+1, currState+1,langExpToken); - } - else if(langExpToken.repeat==RepeatType.STAR) { - addEdge(currState, currState+1,langExpToken); - addEdge(currState+1, currState+1,langExpToken); - //deal with previous edges - Map> prevEdges = incomingEdges[currState]; //could be NULL - for efficiency I allow this - if(prevEdges!=null) { - for(LanguageExpToken prevToken: prevEdges.keySet()) { - for(Integer prevState: prevEdges.get(prevToken)) { - addEdge(prevState, currState+1, prevToken); - } - } - } - //deal with start state - if(startStates.contains(currState)) - startStates.add(currState+1); - } - else if(langExpToken.repeat==RepeatType.Q_MARK) { - addEdge(currState, currState+1,langExpToken); - //deal with previous edges - Map> prevEdges = incomingEdges[currState]; //could be NULL - for efficiency I allow this - if(prevEdges!=null) { - for(LanguageExpToken prevToken: prevEdges.keySet()) { - for(Integer prevState: prevEdges.get(prevToken)) { - addEdge(prevState, currState+1, prevToken); - } - } - } - //deal with start state - if(startStates.contains(currState)) - startStates.add(currState+1); - } - else - addEdge(currState, currState+1,langExpToken); - numOfStates++; - } - } - - private void addEdge(int source, int destination, LanguageExpToken label){ - Map> out = outgoingEdges[source]; - if(out==null) { - out = new HashMap>(); - outgoingEdges[source]=out; - } - Map> in = incomingEdges[destination]; - if(in==null) { - in = new HashMap>(); - incomingEdges[destination]=in; - } - - MapUtils.addToSet(out, label, destination); - MapUtils.addToSet(in, label, source); - } - - private List match(LanguageInfo utterance, int startIndex) { - - List res = new ArrayList(); - - Set currStates = new HashSet(); - currStates.addAll(startStates); - if(currStates.contains(numOfStates-1)) - res.add(new Interval(startIndex, startIndex)); - - for(int i = startIndex; i < utterance.numTokens(); ++i) { - Set newStates = new HashSet(); - for(Integer currState: currStates) { - Map> out = outgoingEdges[currState]; //could be NULL - allowed for efficiency - if(out!=null) { - for(LanguageExpToken langExpToken: out.keySet()) { - if(matchLangExpTokenToLangInfo(langExpToken, utterance, i)) { - newStates.addAll(out.get(langExpToken)); - } - } - } - } - currStates = newStates; - if(currStates.contains(numOfStates-1)) //match - res.add(new Interval(startIndex, i+1)); - } - return res; - } - - private boolean matchLangExpTokenToLangInfo(LanguageExpToken langExpToken, - LanguageInfo info, int infoIndex) { - if(langExpToken.type==TokenType.pos) { - return langExpToken.valuePattern.matcher(info.posTags.get(infoIndex)).matches(); - } - if(langExpToken.type==TokenType.ner) { - return langExpToken.valuePattern.matcher(info.nerTags.get(infoIndex)).matches(); - } - if(langExpToken.type==TokenType.lemma) { - return langExpToken.valuePattern.matcher(info.lemmaTokens.get(infoIndex)).matches(); - } - if(langExpToken.type==TokenType.token) { - return langExpToken.valuePattern.matcher(info.tokens.get(infoIndex)).matches(); - } - throw new RuntimeException("illegal language expression token: " + langExpToken.toLispTree()); - } - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExp.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExp.java deleted file mode 100644 index 2abfe6b..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExp.java +++ /dev/null @@ -1,112 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.ArrayList; -import java.util.List; - -import com.google.common.base.Joiner; - -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.paraphrase.Interval; -import fig.basic.LispTree; - -/** - * A list of language items - * Matching a language expression basically amounts to matching each one of the items and building all possible - * matches - * @author jonathanberant - * - */ -public class LanguageExp { - - private List items = new ArrayList(); - - public LanguageExp(LispTree tree) { - for(int i = 1; i < tree.children.size(); ++i) - items.add(new LangItem(tree.child(i))); - } - - public int itemCount() { return items.size(); } - - public String toString() { - return Joiner.on(' ').join(items); - } - - public List items() { - return items; - } - - public LangItem getItem(int i) { - return items.get(i); - } - - /** - * Matching the entire utterance against the expression - * @param info - */ - public List match(LanguageInfo utterance) { - - //generate the matches of the first item - List currMatches = new ArrayList(); - List firstItemIntervals = items.get(0).match(utterance, 0); - for(Interval interval: firstItemIntervals) { - currMatches.add(new LangExpMatch(interval.start,interval.end)); - } - //go over the rest of the items and match them - for(int i = 1; i < items.size(); ++i) { - LangItem currItem = items.get(i); - List newMatches = new ArrayList(); - for(LangExpMatch currMatch: currMatches) { - List intervals = currItem.match(utterance, currMatch.end()); - for(Interval interval: intervals) { - LangExpMatch newLangExpMatch = new LangExpMatch(currMatch); - newLangExpMatch.addInterval(interval); - newMatches.add(newLangExpMatch); - } - } - currMatches = newMatches; - } - //keep matches that match the entire utterance - List res = new ArrayList(); - for(LangExpMatch currMatch: currMatches) { - if(currMatch.end==utterance.numTokens()) - res.add(currMatch); - } - return res; - } - - public static class LangExpMatch { - - private ArrayList itemMap; - private int end; - private int lastItemMatched=0; - - public LangExpMatch(int start, int end) { - itemMap = new ArrayList(); - itemMap.add(new Interval(start,end)); - this.end = end; - } - - public LangExpMatch(LangExpMatch other) { - itemMap = new ArrayList(); - for(Interval otherInterval: other.itemMap) - itemMap.add(new Interval(otherInterval.start, otherInterval.end)); - - this.end = other.end; - this.lastItemMatched = other.lastItemMatched; - } - - public void addInterval(Interval interval) { - itemMap.add(interval); - lastItemMatched++; - this.end = interval.end; - } - - public String toString() { - return itemMap.toString() + " end: " + end; - } - - public int end() { return end; } - - public Interval get(int index) {return itemMap.get(index); } - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExpToken.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExpToken.java deleted file mode 100644 index eeabebb..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExpToken.java +++ /dev/null @@ -1,82 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.regex.Pattern; - -import fig.basic.LispTree; - -public class LanguageExpToken { - - public enum TokenType {token, ner, pos, lemma} - public enum RepeatType {NONE, Q_MARK, STAR, PLUS} - - public final TokenType type; - public final RepeatType repeat; - public final Pattern valuePattern; - public final String value; - - public LanguageExpToken(String type, String value) { - this.type = parseTokenType(type); - this.valuePattern = Pattern.compile(value.substring(1,value.lastIndexOf(']'))); - this.repeat = parseRepeatType(value.endsWith("]") ? "" : value.substring(value.length()-1)); - this.value=value; - } - - public static TokenType parseTokenType(String type) { - if ("token".equals(type)) return TokenType.token; - if ("lemma".equals(type)) return TokenType.lemma; - if ("pos".equals(type)) return TokenType.pos; - if ("ner".equals(type)) return TokenType.ner; - throw new RuntimeException("Illegal token type: " + type); - } - - public static RepeatType parseRepeatType(String repeat) { - if("".equals(repeat)) return RepeatType.NONE; - if("?".equals(repeat)) return RepeatType.Q_MARK; - if("+".equals(repeat)) return RepeatType.PLUS; - if("*".equals(repeat)) return RepeatType.STAR; - throw new RuntimeException("Illegal repeat type: " + repeat); - } - - public LispTree toLispTree() { - LispTree tree = LispTree.proto.newList(); - tree.addChild(type.toString()); - tree.addChild(valuePattern.toString()); - return tree; - } - - public String toString() { - return toLispTree().toString(); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((repeat == null) ? 0 : repeat.hashCode()); - result = prime * result + ((type == null) ? 0 : type.hashCode()); - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - LanguageExpToken other = (LanguageExpToken) obj; - if (repeat != other.repeat) - return false; - if (type != other.type) - return false; - if (value == null) { - if (other.value != null) - return false; - } else if (!value.equals(other.value)) - return false; - return true; - } - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosRule.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosRule.java deleted file mode 100644 index 56fadaf..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosRule.java +++ /dev/null @@ -1,57 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -public class LemmaPosRule { - - private LemmaPosSequence lhs; - private LemmaPosSequence rhs; - - public LemmaPosRule(LemmaPosSequence lhs, LemmaPosSequence rhs) { - this.lhs = lhs; - this.rhs = rhs; - } - - public boolean isEmpty() { - return lhs.isEmpty() && rhs.isEmpty(); - } - - public LemmaPosRule reverseRule() { - return new LemmaPosRule(rhs, lhs); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((lhs == null) ? 0 : lhs.hashCode()); - result = prime * result + ((rhs == null) ? 0 : rhs.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - LemmaPosRule other = (LemmaPosRule) obj; - if (lhs == null) { - if (other.lhs != null) - return false; - } else if (!lhs.equals(other.lhs)) - return false; - if (rhs == null) { - if (other.rhs != null) - return false; - } else if (!rhs.equals(other.rhs)) - return false; - return true; - } - - public String toString() { - return lhs.toString()+"-->"+rhs.toString(); - } - - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosSequence.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosSequence.java deleted file mode 100644 index bde8f0b..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosSequence.java +++ /dev/null @@ -1,83 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.List; - -import com.google.common.base.Joiner; - -public class LemmaPosSequence { - - private List sequence; - - public LemmaPosSequence(List sequence) { - this.sequence = sequence; - } - - public boolean isEmpty() { - return sequence.size()==0; - } - public String toString() { - return Joiner.on(',').join(sequence); - } - - @Override - public int hashCode() { - return toString().hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - LemmaPosSequence other = (LemmaPosSequence) obj; - if(this.sequence.size()!=other.sequence.size()) - return false; - for(int i = 0; i%s",appInfo.type, - antecedent.phrase(0, antecedent.numTokens()),consequent.phrase(0, consequent.numTokens())); - } - - public static class ApplicationInfo { - public final String type; - public final String value; - - public ApplicationInfo(String type, String value) { - this.type = type; - this.value = value; - } - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplier.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplier.java deleted file mode 100644 index 65b66df..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplier.java +++ /dev/null @@ -1,435 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import com.google.common.base.Joiner; - -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.LanguageInfo.LanguageUtils; -import edu.stanford.nlp.sempre.LanguageInfo.WordInfo; -import edu.stanford.nlp.sempre.paraphrase.Interval; -import edu.stanford.nlp.sempre.paraphrase.ParaphraseFeatureMatcher; -import edu.stanford.nlp.sempre.paraphrase.ParaphraseUtils; -import edu.stanford.nlp.sempre.paraphrase.Aligner.AlignmentStats; -import edu.stanford.nlp.sempre.paraphrase.paralex.PhraseTable; -import edu.stanford.nlp.sempre.paraphrase.rules.LanguageExp.LangExpMatch; -import edu.stanford.nlp.sempre.paraphrase.rules.RuleApplication.ApplicationInfo; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.util.StringUtils; -import fig.basic.IntPair; -import fig.basic.LispTree; -import fig.basic.Option; - -public abstract class RuleApplier { - - public static class Options { - @Option public int verbose=1; - } - public static Options opts = new Options(); - - public static final String DELETE = "Del"; - public static final String DELETE_IN_CONTEXT = "Del_ctxt"; - public static final String DELETE_V_SEM="Del_v_sem"; - public static final String SUBST = "Subst"; - public static final String SYNT_SUBST = "SyntSubst"; - public static final String MOVE = "Move"; - public static final String PHRASE_TABLE = "Pt"; - public static final String RULE = "Rule"; - public static final String INSERT = "Ins"; - public static final String SYNTAX = "Syntax"; - public static final String SYNT_ALIGN = "SyntAlign"; - - public abstract List apply(LanguageInfo antecedent, LanguageInfo target); - public abstract List match(LanguageInfo lInfo); -} - -class SyntacticRule extends RuleApplier { - - protected LanguageExp lhs; - protected List rhs=new ArrayList();; //list of either lhs items or wordinfos for generating the consquent - - public SyntacticRule(String lispTree) { - LispTree tree = LispTree.proto.parseFromString(lispTree); - if(tree.children.size()!=3) - throw new RuntimeException("Number of children in a rule must be three"); - lhs = new LanguageExp(tree.child(1)); - LispTree rhsTree = tree.child(2); - - //now go over the tree - for(int i = 1; i < rhsTree.children.size();++i) - rhs.add(addRhsObject(rhsTree.child(i).value)); - } - - private Object addRhsObject(String value) { - try { - return Integer.parseInt(value); - } - catch(NumberFormatException e) { - String[] tokens = value.split(";;"); //HACK to get word info - return new WordInfo(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4]); - } - } - - public List apply(LanguageInfo antecedent, LanguageInfo target) { - - List res = new LinkedList(); - List matches = lhs.match(antecedent); - - for(LangExpMatch match: matches) { - List applications = generateApplications(antecedent,match, target); - res.addAll(applications); - } - return res; - } - - //TODO reuse code in generateApplications - protected List generateApplications(LanguageInfo antecedent, LangExpMatch match, LanguageInfo target) { - - LanguageInfo res = new LanguageInfo(); - for(Object obj: rhs) { - if(obj instanceof WordInfo) { - res.addWordInfo(((WordInfo) obj)); - } - else if(obj instanceof Integer) { - Interval matchingInterval = match.get(((Integer) obj)); - res.addSpan(antecedent, matchingInterval.start, matchingInterval.end); - } - else - throw new RuntimeException("Rhs should be a list of wordinfo or integer keys to intervals only"); - } - RuleApplication application = new RuleApplication(antecedent, res, new ApplicationInfo(RULE,"")); - application.addFeatures(featurizeParaphraseRule()); - return Collections.singletonList(application); - } - - private FeatureVector featurizeParaphraseRule() { - FeatureVector fv = new FeatureVector(); - fv.add(RULE, lhs+"_"+rhs); - return fv; - } - - public String toString() { - String lhsStr = lhs.toString(); - List items = lhs.items(); - List rhsStr = new ArrayList(); - for(Object obj: rhs) { - if(obj instanceof Integer) - rhsStr.add(items.get((Integer) obj).toString()); - else { - rhsStr.add(((WordInfo) obj).lemma); - } - } - return lhsStr+"->"+Joiner.on(' ').join(rhsStr); - } - - @Override - public List match(LanguageInfo lInfo) { - return lhs.match(lInfo); - } -} - -//In deletion rules we know that the text span specified at index 1 of the match is the one that has been deleted -class DeletionRule extends SyntacticRule { - - public static final Map> verbSemclassMap = VerbSemClassExtractor.getWordToWordCounterMap(); - - public DeletionRule(String expression) { - super(expression); - LispTree tree = LispTree.proto.parseFromString(expression); - LispTree rhsTree = tree.child(2); - if(rhsTree.children.size()!=3) - throw new RuntimeException("In a deletion rules there must be exactly three items on the rhs"); - } - - protected List generateApplications(LanguageInfo antecedent, LangExpMatch match, LanguageInfo target) { - - LanguageInfo res = new LanguageInfo(); - for(Object obj: rhs) { - if(obj instanceof WordInfo) { - res.addWordInfo(((WordInfo) obj)); - } - else if(obj instanceof Integer) { - Interval matchingInterval = match.get(((Integer) obj)); - res.addSpan(antecedent, matchingInterval.start, matchingInterval.end); - } - else - throw new RuntimeException("Rhs should be a list of wordinfo or integer keys to intervals only"); - } - RuleApplication application = new RuleApplication(antecedent, res,new ApplicationInfo(DELETE, - antecedent.lemmaPhrase(match.get(1).start, match.get(1).end))); - application.addFeatures(featurizeDeletedSpan(new Interval(match.get(1).start,match.get(1).end),antecedent)); - return Collections.singletonList(application); - } - - private FeatureVector featurizeDeletedSpan(Interval span, LanguageInfo utterance) { - - List spanProperties = utterance.getSpanProperties(span.start,span.end); - FeatureVector res = new FeatureVector(); - - for(String spanProperty: spanProperties) { - if(ParaphraseFeatureMatcher.containsDomain(DELETE)) - res.add(DELETE,"match: " + spanProperty); - if(ParaphraseFeatureMatcher.containsDomain(DELETE_IN_CONTEXT)) { - if(span.start>0) { - List lcProperties = utterance.getSpanProperties(span.start-1,span.start); - for(String lcProperty: lcProperties) { - res.add(DELETE_IN_CONTEXT,"match_" + spanProperty+",lc_"+lcProperty); //properties of deleted match - } - } - if(span.end rcProperties = utterance.getSpanProperties(span.end,span.end+1); - for(String rcProperty: rcProperties) { - res.add(DELETE_IN_CONTEXT,"match_" + spanProperty+",rc_"+rcProperty); //properties of deleted match - } - } - } - } - //verb semclass features - if(ParaphraseFeatureMatcher.containsDomain(DELETE_V_SEM)) { - String lemmaTokens = utterance.lemmaPhrase(span.start, span.end); - Counter cooccurringWords = verbSemclassMap.get(lemmaTokens); - if(cooccurringWords!=null) { - for(String lemma: utterance.lemmaTokens) { - if(cooccurringWords.containsKey(lemma)) { - res.add(DELETE_V_SEM,"match="+lemmaTokens+",context="+lemma); - res.add(DELETE_V_SEM,"sim",Math.log(cooccurringWords.getCount(lemma)+1)); - } - } - } - } - return res; - } -} - -class SubstitutionRule extends SyntacticRule { - - public SubstitutionRule(String expression) { - super(expression); - LispTree tree = LispTree.proto.parseFromString(expression); - LispTree rhsTree = tree.child(2); - if(rhsTree.children.size()!=3) - throw new RuntimeException("In a substitution rules there must be exactly three items on the rhs"); - } - - protected List generateApplications(LanguageInfo antecedent, LangExpMatch match, LanguageInfo target) { - - List res = new LinkedList(); - List targetIntervals = generateConsequentIntervals(antecedent,match,target); //finding all possible substitutions in the target - for(Interval targetInterval: targetIntervals) { - LanguageInfo consequent = new LanguageInfo(); - for(int i = 0; i < 3; ++i) { - if(i==0 || i==2) { - Interval matchingInterval = match.get(i); - consequent.addSpan(antecedent, matchingInterval.start, matchingInterval.end); - } - else { //i==1 - consequent.addSpan(target, targetInterval.start, targetInterval.end); - } - } - RuleApplication application = new RuleApplication(antecedent, consequent, - new ApplicationInfo(SUBST, antecedent.lemmaPhrase(match.get(1).start, match.get(1).end)+"-->"+ - target.lemmaPhrase(targetInterval.start, targetInterval.end))); - application.addFeatures(featurizeSubstitutedSpan(match.get(1),targetInterval,antecedent,target)); - res.add(application); - } - return res; - } - - private FeatureVector featurizeSubstitutedSpan(Interval antecedentInterval, - Interval targetInterval, LanguageInfo antecedent, - LanguageInfo target) { - - FeatureVector res = new FeatureVector(); - String antecedentLemmas = antecedent.lemmaPhrase(antecedentInterval.start, antecedentInterval.end); - String targetLemmas = target.lemmaPhrase(targetInterval.start, targetInterval.end); - - if(ParaphraseFeatureMatcher.containsDomain(SUBST)) { - int editDist = StringUtils.editDistance(antecedentLemmas, targetLemmas); - boolean prefix = antecedentLemmas.startsWith(targetLemmas) || targetLemmas.startsWith(antecedentLemmas); - res.add(SUBST, "l="+antecedentLemmas+",r="+targetLemmas); - res.add(SUBST, "l_pos="+antecedent.posSeq(antecedentInterval.start, antecedentInterval.end)+ - ",r_pos="+target.posSeq(targetInterval.start, targetInterval.end)); - if(editDist==0) - res.add(SUBST, "dist=0"); - else if(editDist<4) - res.add(SUBST, "dist="+editDist); - else - res.add(SUBST, "dist>3"); - if(prefix) - res.add(SUBST, "prefix"); - } - return res; - } - - /** - * Generate the consequents subject to constraints such as - * (a) only generate consequents from the same pos - * (b) don't generate an identity mapping - * @param antecedent - * @param match - * @param target - * @return - */ - private List generateConsequentIntervals(LanguageInfo antecedent, - LangExpMatch match, LanguageInfo target) { - - List res = new LinkedList(); - - String antecedentPos = antecedent.posTags.get(match.get(1).start); //we know in a substitution rule the antecedent interval is in index 1 - String antecedentLemma = antecedent.lemmaPhrase(match.get(1).start,match.get(1).end); - //span 1 candidates - for(int i = 0; i < target.numTokens(); ++i) { - String targetPos = target.posTags.get(i); - String targetLemma = target.lemmaPhrase(i, i+1); - if(ParaphraseUtils.posCompatible(antecedentPos, targetPos) && !antecedentLemma.equals(targetLemma)) { - res.add(new Interval(i,i+1)); - } - } - //span 2 candidates - if(LanguageUtils.isNN(antecedentPos)) { - for(int i = 0; i < target.numTokens()-1; ++i) { - String targetLemma = target.lemmaPhrase(i, i+2); - if(LanguageUtils.isNN(target.posTags.get(i)) && LanguageUtils.isNN(target.posTags.get(i+1)) && !antecedentLemma.equals(targetLemma)) - res.add(new Interval(i,i+2)); - } - } - return res; - } -} - -class MoveRule extends SyntacticRule { - - public MoveRule(String expression) { - super(expression); - LispTree tree = LispTree.proto.parseFromString(expression); - LispTree rhsTree = tree.child(2); - if(rhsTree.children.size()!=3) - throw new RuntimeException("In a substitution rules there must be exactly three items on the rhs"); - } - - /** - * Move any phrase that exists in the target - */ - protected List generateApplications(LanguageInfo antecedent, LangExpMatch match, LanguageInfo target) { - - List res = new LinkedList(); - Interval matchedInterval = match.get(1); - String movedPhrase = antecedent.lemmaPhrase(matchedInterval.start, matchedInterval.end); //we know that 1 is the index moved - if(!target.lemmaPhrase(0, target.numTokens()).contains(movedPhrase)) //don't move things that are not in the target - return res; - - for(int i = 0; i < antecedent.numTokens(); ++i) { - LanguageInfo consequent = new LanguageInfo(); - if(i>=matchedInterval.start && i <=matchedInterval.end) continue; - if(imatchedInterval.end) { - consequent.addSpan(antecedent, 0, matchedInterval.start); - consequent.addSpan(antecedent, matchedInterval.end,i); - consequent.addSpan(antecedent, matchedInterval.start,matchedInterval.end); - consequent.addSpan(antecedent, i,antecedent.numTokens()); - } - RuleApplication application = new RuleApplication(antecedent, consequent, - new ApplicationInfo(MOVE,antecedent.lemmaPhrase(matchedInterval.start, matchedInterval.end))); - application.addFeatures(featurizeMovedSpan(antecedent,matchedInterval)); - res.add(application); - } - return res; - } - - private FeatureVector featurizeMovedSpan(LanguageInfo antecedent, - Interval matchedInterval) { - FeatureVector res = new FeatureVector(); - - if(ParaphraseFeatureMatcher.containsDomain(MOVE)) { - res.add(MOVE,"lemmas="+antecedent.lemmaPhrase(matchedInterval.start, matchedInterval.end)); - res.add(MOVE,"pos="+antecedent.posSeq(matchedInterval.start, matchedInterval.end)); - res.add(MOVE,"ner="+antecedent.nerSeq(matchedInterval.start, matchedInterval.end)); - } - return res; - } -} - -class PhraseTableRule extends SyntacticRule { - - PhraseTable phraseTable; - - public PhraseTableRule(String expression) { - super(expression); - LispTree tree = LispTree.proto.parseFromString(expression); - LispTree rhsTree = tree.child(2); - if(rhsTree.children.size()!=3) - throw new RuntimeException("In a phrasetable rule there must be exactly three items on the rhs"); - - if(phraseTable==null) - phraseTable = PhraseTable.getSingleton(); - } - - - - protected List generateApplications(LanguageInfo antecedent, LangExpMatch match, LanguageInfo target) { - - List res = new LinkedList(); - //get LHS phrase - guarateed that 1 is the index of the match - int matchStart = match.get(1).start; - int matchEnd = match.get(1).end; - - String lhsPhrase = antecedent.lemmaPhrase(matchStart, matchEnd); - Map rhsCandidates = phraseTable.get(lhsPhrase); - if(rhsCandidates==null) { - return res; - } - - //get RHS - Map rhsLemmaSpans = target.getLemmaSpans(); - for(String rhsLemmaSpan: rhsLemmaSpans.keySet()) { - if(rhsCandidates.containsKey(rhsLemmaSpan)) { - - int targetStart = rhsLemmaSpans.get(rhsLemmaSpan).first; - int targetEnd = rhsLemmaSpans.get(rhsLemmaSpan).second; - LanguageInfo consequent = generateConsequent(antecedent,matchStart,matchEnd, - target, targetStart,targetEnd); - RuleApplication application = new RuleApplication(antecedent, consequent, - new ApplicationInfo(PHRASE_TABLE, lhsPhrase+"-->"+rhsLemmaSpan)); - application.addFeatures(featureizePhraseTablespan(lhsPhrase,rhsLemmaSpan,rhsCandidates.get(rhsLemmaSpan), - antecedent,matchStart,matchEnd,target,targetStart,targetEnd)); - //do not do an identity transformation - res.add(application); - } - } - return res; - } - - private FeatureVector featureizePhraseTablespan(String lemmaPhrase, - String rhsLemmaSpan, AlignmentStats count, LanguageInfo antecedent, int matchStart, - int matchEnd, LanguageInfo target, int targetStart, int targetEnd) { - - FeatureVector res = new FeatureVector(); - if(ParaphraseFeatureMatcher.containsDomain(PHRASE_TABLE)) { - res.add(SUBST, "l="+lemmaPhrase+",r="+rhsLemmaSpan); - res.add(SUBST, "l_pos="+antecedent.posSeq(matchStart, matchEnd)+",r_pos="+target.posSeq(targetStart, targetEnd)); - //res.add(PHRASE_TABLE, "PtScore", Math.log(count+1)); - } - return res; - } - - private LanguageInfo generateConsequent(LanguageInfo antecedent, - int matchStart, int matchEnd, LanguageInfo target, int targetStart, - int targetEnd) { - LanguageInfo res = new LanguageInfo(); - res.addSpan(antecedent, 0, matchStart); - res.addSpan(target, targetStart, targetEnd); - res.addSpan(antecedent, matchEnd, antecedent.numTokens()); - return res; - } - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/Rulebase.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/Rulebase.java deleted file mode 100644 index cba6aa8..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/Rulebase.java +++ /dev/null @@ -1,202 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; - -import com.google.common.base.Joiner; - -import fig.basic.Option; -import fig.basic.LogInfo; - -public class Rulebase { - - public static class Options { - @Option(gloss = "What rule types to use") - public Set ruleTypes; - } - public static Options opts = new Options(); - - List rules; - - public static final String NULL_PHRASE = "(item (token [.*]*))"; - public static final String PHRASE = "(item (token [.*]+))"; - public static final String TOKEN = "(item (token [.*]))"; - - //general delete and substitute rules - public static final String DELETE_TOKEN = "(rule (expression "+ PHRASE + TOKEN + PHRASE + ") (rhs 0 2))"; - public static final String DELETE_PHRASE = "(rule (expression "+ PHRASE + PHRASE + PHRASE + ") (rhs 0 2))"; - public static final String DELETE_TWO_NOUNS = "(rule (expression "+ PHRASE + "(item (pos [NN.*]) (pos [NN.*]))" + PHRASE + ") (rhs 0 2))"; - public static final String SUBSTITUTE_TOKEN = "(rule (expression "+ PHRASE + TOKEN + PHRASE + ") (rhs 0 2))"; - public static final String SUBSTITUTE_PHRASE = "(rule (expression "+ PHRASE + PHRASE + PHRASE + ") (rhs 0 2))"; - public static final String SUBSTITUTE_TWO_NOUNS = "(rule (expression "+ PHRASE + "(item (pos [NN|NNS]) (pos [NN|NNS]))" + PHRASE + ") (rhs 0 2))"; - public static final String MOVE_PHRASE = "(rule (expression "+ PHRASE + PHRASE + PHRASE + ") (rhs 0 2))"; - - //specific rules - public static final String DELETE_DT = "(rule (expression (item (token [.*]*)) (item (pos [DT])) (item (token [.*]*))) (rhs 0 2))"; - public static final String DELETE_JJ = "(rule (expression (item (token [.*]*)) (item (pos [JJ])) (item (token [.*]*))) (rhs 0 2))"; - public static final String DELETE_RB = "(rule (expression (item (token [.*]*)) (item (pos [RB])) (item (token [.*]*))) (rhs 0 2))"; - public static final String DELETE_PRP = "(rule (expression (item (token [.*]*)) (item (pos [PRP])) (item (token [.*]*))) (rhs 0 2))"; - public static final String DELETE_DATE_1 = "(rule (expression (item (token [.*]*)) (item (pos [IN]) (ner [DATE])) (item (token [.*]*))) (rhs 0 2))"; - public static final String DELETE_DATE_2 = "(rule (expression (item (token [.*]*)) (item (ner [DATE])) (item (token [.*]*))) (rhs 0 2))"; - public static final String DELETE_AUX = "(rule (expression (item (token [.*]*)) (item (pos [VBD-AUX])) (item (token [.*]*))) (rhs 0 2))"; - //delete phrases - public static final String NP = "(item (pos [DT]?)) (item (pos [JJ.*]*)) (item (pos [NN.*]+))"; - public static final String VERB = "(item (pos [VB.*])) (item (pos [RP|IN|TO]))"; - public static final String ENTITY = "(item (pos [NNP.*]+))"; - - public static final String DELETE_NP = "(rule (expression " + NULL_PHRASE + NP + NULL_PHRASE + ") (rhs 0 4))"; - public static final String DELETE_PP = "(rule (expression " + NULL_PHRASE + " (item (pos [IN|TO])) " +NP + NULL_PHRASE + ") (rhs 0 5))"; - - //nominalization - public static final String NOM1 = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression (item (lemma [what]))", - NP, - VERB, - ENTITY, - "(item (token [.*])))", - "(rhs 0 be;;be;;VBZ;;O;;O the;;the;;DT;;O;;OO 1 2 3 of;;of;;IN;;O;;O 6))" - })); - - public static final String NOM2 = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression (item (lemma [what]))", - NP, - "(item (lemma [do]))", - ENTITY, - VERB, - "(item (token [.*])))", - "(rhs 0 be;;be;;VBZ;;O;;O the;;the;;DT;;O;;OO 1 2 3 of;;of;;IN;;O;;O 5))" - })); - - //passive - public static final String PASSIVE = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression", - NULL_PHRASE, - VERB, - "(item (token [by]))", - ENTITY, - NULL_PHRASE+")", - "(rhs 0 4 1 2 5))" - })); - - public static final String POSSESSIVE_1 = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression", - NULL_PHRASE, - NP, - "(item (token [of]))", - NP, - NULL_PHRASE+")", - "(rhs 0 5 6 7 1 2 3 8))" - })); - - public static final String POSSESSIVE_2 = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression", - NULL_PHRASE, - NP, - "(item (token [of]))", - NP, - NULL_PHRASE+")", - "(rhs 0 5 6 7 's;;'s;;POS;;O;;O 1 2 3 8))" - })); - //same syntactic rules but omitting the context - just the match -//nominalization - public static final String NOM1_ALIGN = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression (item (lemma [what]))", - NP, - VERB, - ENTITY, - "(item (token [.*])))", - "(rhs be;;be;;VBZ;;O;;O the;;the;;DT;;O;;OO 1 2 3 of;;of;;IN;;O;;O))" - })); - - public static final String NOM2_ALIGN = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression (item (lemma [what]))", - NP, - "(item (lemma [do]))", - ENTITY, - VERB, - "(item (token [.*])))", - "(rhs be;;be;;VBZ;;O;;O the;;the;;DT;;O;;OO 1 2 3 of;;of;;IN;;O;;O))" - })); - - //passive - public static final String PASSIVE_ALIGN = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression", - NULL_PHRASE, - VERB, - "(item (token [by]))", - ENTITY, - NULL_PHRASE+")", - "(rhs 4 1 2))" - })); - - public static final String POSSESSIVE_1_ALIGN = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression", - NULL_PHRASE, - NP, - "(item (token [of]))", - NP, - NULL_PHRASE+")", - "(rhs 5 6 7 1 2 3))" - })); - - public static final String POSSESSIVE_2_ALIGN = Joiner.on(' ').join(Arrays.asList(new String[] - {"(rule (expression", - NULL_PHRASE, - NP, - "(item (token [of]))", - NP, - NULL_PHRASE+")", - "(rhs 5 6 7 's;;'s;;POS;;O;;O 1 2 3))" - })); - - - public Rulebase() { - rules = new ArrayList(); - - if(opts.ruleTypes.contains(RuleApplier.DELETE)) { - rules.add(new DeletionRule(DELETE_TOKEN)); - rules.add(new DeletionRule(DELETE_TWO_NOUNS)); - } - if(opts.ruleTypes.contains(RuleApplier.SUBST)) { - rules.add(new SubstitutionRule(SUBSTITUTE_TOKEN)); - rules.add(new SubstitutionRule(SUBSTITUTE_TWO_NOUNS)); - } - if(opts.ruleTypes.contains(RuleApplier.MOVE)) - rules.add(new MoveRule(MOVE_PHRASE)); - if(opts.ruleTypes.contains(RuleApplier.SYNTAX)) { - LogInfo.log("Adding syntax rule"); - rules.add(new SyntacticRule(DELETE_PP)); - rules.add(new SyntacticRule(DELETE_DT)); - rules.add(new SyntacticRule(DELETE_JJ)); - rules.add(new SyntacticRule(DELETE_RB)); - rules.add(new SyntacticRule(DELETE_PRP)); - rules.add(new SyntacticRule(DELETE_AUX)); - rules.add(new SyntacticRule(NOM1)); - rules.add(new SyntacticRule(NOM2)); - rules.add(new SyntacticRule(PASSIVE)); - rules.add(new SyntacticRule(POSSESSIVE_1)); - rules.add(new SyntacticRule(POSSESSIVE_2)); - } - if(opts.ruleTypes.contains(RuleApplier.PHRASE_TABLE)) { - rules.add(new PhraseTableRule(SUBSTITUTE_PHRASE)); - } - if(opts.ruleTypes.contains(RuleApplier.SYNT_SUBST)) { - LogInfo.log("Adding syntactic rules"); - rules.add(new SyntacticRuleSet()); - } - if(opts.ruleTypes.contains(RuleApplier.SYNT_ALIGN)) { - LogInfo.log("Adding syntactic rules for alignment"); - rules.add(new SyntacticRule(NOM1_ALIGN)); - rules.add(new SyntacticRule(NOM2_ALIGN)); - rules.add(new SyntacticRule(PASSIVE_ALIGN)); - rules.add(new SyntacticRule(POSSESSIVE_1_ALIGN)); - rules.add(new SyntacticRule(POSSESSIVE_2_ALIGN)); - } - } - - public List getRules() { - return rules; - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/SubstitutionRuleExtractor.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/SubstitutionRuleExtractor.java deleted file mode 100644 index 4971c0e..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/SubstitutionRuleExtractor.java +++ /dev/null @@ -1,83 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.io.BufferedReader; -import java.io.IOException; -import java.util.List; - -import com.google.common.collect.Lists; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.Json; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.paraphrase.ParaphraseDataset; -import edu.stanford.nlp.sempre.paraphrase.ParaphraseExample; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.stats.Counters; -import fig.basic.LogInfo; -import fig.basic.Option; - -public class SubstitutionRuleExtractor { - - public static class Options { - @Option public String dataFile; - @Option public int verbose=1; - @Option public int maxExamples=Integer.MAX_VALUE; - } - public static Options opts = new Options(); - - private Counter ruleBase = new ClassicCounter(); - - public SubstitutionRuleExtractor() { - //don't do ner - LanguageInfo.opts.annotators = Lists.newArrayList("tokenize", "ssplit", "pos", "lemma"); - } - public void extractRules() throws IOException { - LogInfo.begin_track_printAll("Extracting substitution rules:"); - int i = 0; - BufferedReader reader = IOUtils.getBufferedFileReader(opts.dataFile); - ParaphraseDataset dataset = Json.readValueHard(reader, ParaphraseDataset.class); - for(ParaphraseExample ex: dataset.paraphraseExamples("train")) { - extractRule(ex); - if(++i % 10000 == 0) - LogInfo.logs("Number of examples gone through: %s",i); - if(i>=opts.maxExamples) - break; - } - LogInfo.end_track(); - } - - private void extractRule(ParaphraseExample ex) { - - ParaphraseAlignment alignment = ex.align(); - //first check if we can extract rule - if(alignment.isLegalSingleGap()) { - LemmaPosRule rule = ex.getRule(alignment.getSourceInterval(),alignment.getTargetInterval()); - if(rule.isEmpty() && opts.verbose>=1) { - LogInfo.logs("SubstitutionRuleExtractor.extractRule: empty rule for source=%s, target=%s",ex.source,ex.target); - } - if(opts.verbose>=3) - LogInfo.logs("SubstitutionRuleExtractor.extract: example source=%s, example target=%s rule=%s",ex.source,ex.target,rule); - ruleBase.incrementCount(rule); - ruleBase.incrementCount(rule.reverseRule()); - } - } - - public void log() { - LogInfo.begin_track("Logging rules"); - List sorted = Counters.toSortedList(ruleBase, false); - for(LemmaPosRule rule: sorted) { - LogInfo.logs("SubstitutionRuleExtractor.log: rule:%s\tcount:%s",rule,ruleBase.getCount(rule)); - } - LogInfo.end_track(); - } - - public static void main(String[] args) throws IOException { - - SubstitutionRuleExtractor.opts.dataFile="/Users/jonathanberant/Research/temp/webquestions.train"; - SubstitutionRuleExtractor extractor = new SubstitutionRuleExtractor(); - extractor.extractRules(); - extractor.log(); - - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/SyntacticRuleSet.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/SyntacticRuleSet.java deleted file mode 100644 index e57e2d5..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/SyntacticRuleSet.java +++ /dev/null @@ -1,322 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.google.common.base.Joiner; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.FeatureVector; -import edu.stanford.nlp.sempre.LanguageInfo; -import edu.stanford.nlp.sempre.LanguageInfo.LanguageUtils; -import edu.stanford.nlp.sempre.LanguageInfo.WordInfo; -import edu.stanford.nlp.sempre.paraphrase.rules.LanguageExp.LangExpMatch; -import edu.stanford.nlp.sempre.paraphrase.rules.RuleApplication.ApplicationInfo; -import fig.basic.LogInfo; -import fig.basic.Option; -import fig.basic.Pair; - -/** - * Storing and matching efficiently a large number of syntactic rules - * We do not use the regexp mechanism since in the trie we want to use a map - * to find the next edges relevant and not go over all outgoing edges and match the regexp - * over them - * @author jonathanberant - */ - -public class SyntacticRuleSet extends RuleApplier { - - public static class Options { - @Option(gloss= "Path to syntactic rule set") public String rulesetPath="lib/paralex/syntactic-rules.retagged.sorted.txt"; - @Option(gloss= "Verbosity") public int verbose=0; - } - public static Options opts = new Options(); - - Trie trie = new Trie(); - - public SyntacticRuleSet() { - LogInfo.begin_track("Loading syntactic rules"); - loadRuleset(); - LogInfo.end_track(); - } - - /** - * loading the rules we filter - * (a) things with punctuations - * (b) things with derivations - * (c) things where pos tags don't match? - * (d) count threshold? - */ - private void loadRuleset() { - int count=0; - for(String line: IOUtils.readLines(opts.rulesetPath)) { - if(validRule(line)) { - count++; - SubstitutableSyntacticRule rule = parseRule(line); - if(opts.verbose>=3) - LogInfo.logs("loadRuleSet: uploaded rule=%s",rule); - trie.add(parseRule(line)); - } - } - LogInfo.logs("Number of valid rules=%s",count); - } - - private boolean validRule(String line) { - if(line.contains("DER")) - return false; - if(line.contains("._")) - return false; - String[] tokens = line.split("\t"); - double count = Double.parseDouble(tokens[2]); - if(count<20.0) - return false; - String[] lhsTokens = tokens[0].split("\\s+"); - String[] rhsTokens = tokens[1].split("\\s+"); - Set lhsVars = new HashSet(); - Set rhsVars = new HashSet(); - List rhsVarList = new ArrayList(); - for(String lhsToken: lhsTokens) { - if(Character.isDigit(lhsToken.charAt(0))) - return false; - if(lhsToken.indexOf('_')!=-1) - lhsVars.add(Integer.parseInt(lhsToken.split("_")[1])); - } - for(String rhsToken: rhsTokens) { - if(Character.isDigit(rhsToken.charAt(0))) - return false; - if(rhsToken.indexOf('_')!=-1) { - rhsVars.add(Integer.parseInt(rhsToken.split("_")[1])); - rhsVarList.add(Integer.parseInt(rhsToken.split("_")[1])); - } - } -// if(!SetUtils.isEqualSet(lhsVars, rhsVars)) -// return false; - if(lhsVars.size()<2 || rhsVars.size()<2) //extreme rule - return false; - for(int i = 0; i < rhsVarList.size()-1;++i) { - if(rhsVarList.get(i)>rhsVarList.get(i+1)) - return true; - } - return false; - // return true; - } - - //TODO very hacky - private SubstitutableSyntacticRule parseRule(String line) { - - String[] tokens = line.split("\t"); - double count = Double.parseDouble(tokens[2]); - List lhs = new ArrayList(); - - String[] lhsTokens = tokens[0].split("\\s+"); - int[] varNameToPositionMap =new int[lhsTokens.length]; - String[] rhsTokens = tokens[1].split("\\s+"); - int[] mapping = new int[rhsTokens.length]; - Arrays.fill(mapping, -1); - Arrays.fill(varNameToPositionMap, -1); - WordInfo[] rhs = new WordInfo[rhsTokens.length]; - //generate lhs - for(int i = 0; i < lhsTokens.length; ++i) { - String lhsToken = lhsTokens[i]; - if(lhsToken.indexOf('_')!=-1) { - String[] posAndVarName = lhsToken.split("_"); - lhs.add(new LanguageExpToken("pos", "["+posAndVarName[0]+"]")); - varNameToPositionMap[Integer.parseInt(posAndVarName[1])]=i; - } - else { - String[] wordInfoParts = lhsToken.split("\\|\\|"); - lhs.add(new LanguageExpToken("lemma","["+wordInfoParts[0]+"]")); - } - } - //generate mapping and rhs - for(int i = 0; i < rhsTokens.length; ++i) { - String rhsToken = rhsTokens[i]; - if(rhsToken.indexOf('_')!=-1) { - String[] posAndVarName = rhsToken.split("_"); - mapping[i]=varNameToPositionMap[Integer.parseInt(posAndVarName[1])]; - } - else { - String[] wordInfoParts = rhsToken.split("\\|\\|"); - rhs[i]=new WordInfo(wordInfoParts[0], wordInfoParts[0], wordInfoParts[1], wordInfoParts[2], "O"); - } - } - return new SubstitutableSyntacticRule(lhs, rhs, mapping, count); - } - - @Override - public List apply(LanguageInfo antecedent, - LanguageInfo target) { - - List res = new ArrayList(); - //go over all spans - for(int i = 0; i < antecedent.numTokens(); ++i) { - List currentNodes = new ArrayList(); - currentNodes.add(trie); - for(int span=1; span<=5 && i+span <= antecedent.numTokens(); ++span) { - if(i==1 && span==1) - System.out.print(""); - List nextNodes = new ArrayList(); - Pair lemmaPair = new Pair("lemma",antecedent.lemmaTokens.get(i+span-1)); - Pair posPair = new Pair("pos",LanguageUtils.getCanonicalPos(antecedent.posTags.get(i+span-1))); - Pair nerPair = new Pair("ner",antecedent.nerTags.get(i+span-1)); - //add to next nodes all the tries we can reach with the current word - addReachableTries(currentNodes, nextNodes, lemmaPair, posPair, nerPair); - //now we can apply all of the rules - generateMatchingRhsApplications(antecedent, target, res, i, span, nextNodes); - //we set current nodes to next nodes for the next round - currentNodes = nextNodes; - } - } - return res; - } - - private void generateMatchingRhsApplications(LanguageInfo antecedent, LanguageInfo target, - List res, int i, int span, List nextNodes) { - - Set generatedRhsMatches = new HashSet(); - for(Trie nextNode: nextNodes) { - for(SubstitutableSyntacticRule rule: nextNode.rules) { - List rhsMatch = rule.generateRhsLemmas(antecedent, i); - if(target.matchLemmas(rhsMatch)) { - String rhsMatchPhrase = LanguageUtils.getLemmaPhrase(rhsMatch); - if(!generatedRhsMatches.contains(rhsMatchPhrase)) { - res.add(generateApplications(antecedent, i, i+span, rhsMatch, rule)); - } - } - } - } - } - - private void addReachableTries(List currentNodes, List nextNodes, - Pair lemmaPair, Pair posPair, - Pair nerPair) { - for(Trie currentNode: currentNodes) { - addNextTrie(nextNodes, lemmaPair, currentNode); - addNextTrie(nextNodes, posPair, currentNode); - addNextTrie(nextNodes, nerPair, currentNode); - } - } - - private void addNextTrie(List nextNodes, Pair lemmaPair, - Trie currentNode) { - Trie nextTrie = currentNode.next(lemmaPair); - if(nextTrie!=null) - nextNodes.add(nextTrie); - } - - private RuleApplication generateApplications(LanguageInfo antecedent, int i, int j, List rhsMatch, SubstitutableSyntacticRule rule) { - LanguageInfo consequent = new LanguageInfo(); - consequent.addSpan(antecedent, 0, i); - consequent.addWordInfos(rhsMatch); - consequent.addSpan(antecedent, j, antecedent.numTokens()); - RuleApplication application = new RuleApplication(antecedent, consequent, new ApplicationInfo(SYNT_SUBST, rule.toString())); - FeatureVector fv = new FeatureVector(); - fv.add(SYNT_SUBST, rule.toString()); - fv.add(SYNT_SUBST, "score" ,rule.count); - application.addFeatures(fv); - if(opts.verbose>0) - LogInfo.logs("antecedent=%s, consequent=%s, rule=%s",antecedent.tokens,consequent.tokens,rule); - return application; - } - - class SubstitutableSyntacticRule { - - final List lhs; - final WordInfo[] rhs; - final int[] mapping; - final double count; - - public SubstitutableSyntacticRule(List lhs, WordInfo[] rhs, - int[] mapping, double count) { - this.lhs=lhs; - this.rhs=rhs; - this.mapping=mapping; - this.count = count; - } - - public List generateRhsLemmas(LanguageInfo antecedent, int start) { - List res = new ArrayList(); - for(int i = 0; i < rhs.length; ++i) { - if(rhs[i]!=null) - res.add(rhs[i]); - else - res.add(antecedent.getWordInfo(start+mapping[i])); - } - return res; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - for(int i = 0; i < rhs.length; ++i) { - if(rhs[i]==null) - sb.append(mapping[i]+" "); - else - sb.append(rhs[i].toString()+" "); - } - return Joiner.on(' ').join(lhs)+"-->"+sb.toString()+"("+count+")"; - } - } - - - class Trie { - ArrayList rules = new ArrayList(); - HashMap, Trie> children = new HashMap, Trie>(); - - Trie next(Pair pair) { - return children.get(pair); - } - - void add(SubstitutableSyntacticRule rule) { add(rule, 0); } - private void add(SubstitutableSyntacticRule rule, int i) { - - if(i == rule.lhs.size()) { - rules.add(rule); - return; - } - - LanguageExpToken langToken = rule.lhs.get(i); - Trie child = children.get(convertLangItemToPair(langToken)); - if (child == null) { - children.put(convertLangItemToPair(langToken), - child = new Trie()); - } - child.add(rule, i + 1); - } - - private Pair convertLangItemToPair(LanguageExpToken langToken) { - String value = langToken.value.substring(1,langToken.value.lastIndexOf(']')); - return Pair.newPair(langToken.type.toString(),value); - } - } - - public static void main(String[] args) { - opts.verbose=3; - SyntacticRuleSet.opts.rulesetPath="/Users/jonathanberant/Research/temp/syntactic-rules.retagged.sorted.txt"; - SyntacticRuleSet srt = new SyntacticRuleSet(); - LanguageInfo antecedent = new LanguageInfo(); - antecedent.addWordInfo(new WordInfo("where", "where", "WDT", "O", "O")); - antecedent.addWordInfo(new WordInfo("was", "be", "VBD", "O", "O")); - antecedent.addWordInfo(new WordInfo("obama", "obama", "NNP", "PERSON", "O")); - antecedent.addWordInfo(new WordInfo("birth", "birth", "NN", "O", "O")); - antecedent.addWordInfo(new WordInfo("place", "place", "NN", "O", "O")); - - LanguageInfo target = new LanguageInfo(); - target.addWordInfo(new WordInfo("where", "where", "WDT", "O", "O")); - target.addWordInfo(new WordInfo("was", "be", "VBD", "O", "O")); - target.addWordInfo(new WordInfo("obama", "obama", "NNP", "PERSON", "O")); - target.addWordInfo(new WordInfo("'s", "'s", "POS", "O", "O")); - target.addWordInfo(new WordInfo("place", "place", "NN", "O", "O")); - target.addWordInfo(new WordInfo("of", "of", "IN", "O", "O")); - target.addWordInfo(new WordInfo("birth", "birth", "NN", "O", "O")); - srt.apply(antecedent, target); - } - - @Override - public List match(LanguageInfo lInfo) { - throw new RuntimeException("Unsupoorted method"); - } -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassExtractor.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassExtractor.java deleted file mode 100644 index f1378fa..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassExtractor.java +++ /dev/null @@ -1,131 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import edu.stanford.nlp.io.IOUtils; -import edu.stanford.nlp.sempre.Dataset; -import edu.stanford.nlp.sempre.Example; -import edu.stanford.nlp.sempre.paraphrase.paralex.ParalexQuestionReader; -import edu.stanford.nlp.sempre.paraphrase.rules.VerbSemClassMatcher.VerbSemClassMatch; -import edu.stanford.nlp.stats.ClassicCounter; -import edu.stanford.nlp.stats.Counter; -import edu.stanford.nlp.stats.Counters; -import fig.basic.LogInfo; -import fig.basic.MapUtils; - -public class VerbSemClassExtractor { - - public static class Options { - public String verbSemclassFile = "lib/trans/resources/verb-semclass.txt"; - } - public static Options opts = new Options(); - - private Map> verbToObject= new HashMap>(); - private Map> verbToSubject = new HashMap>(); - private ParalexQuestionReader reader; - private VerbSemClassMatcher matcher = VerbSemClassMatcher.getInstance(); - private Dataset dataset; //webQ dataset - - public VerbSemClassExtractor() throws IOException { - reader = new ParalexQuestionReader(); - dataset = new Dataset(); - Dataset.opts.trainFrac=1.0; - Dataset.opts.devFrac=0.0; - dataset.read(); - } - - public void extract() throws IOException { - extractParalex(); - extractWebQ(); - log(); - } - - private void extractWebQ() { - for(Example example: dataset.examples("train")) { - VerbSemClassMatch match = matcher.match(example.utterance); - if(match!=null) { - LogInfo.logs("Sentence: %s, match: %s",example.utterance,match); - if(match.isSubj) - addMatch(match,verbToSubject); - else - addMatch(match,verbToObject); - } - } - } - - private void extractParalex() throws IOException { - String sentence; - int i = 0; - while((sentence = reader.next())!=null) { - VerbSemClassMatch match = matcher.match(sentence); - if(match!=null) { - // LogInfo.logs("Sentence: %s, match: %s",sentence,match); - if(match.isSubj) - addMatch(match,verbToSubject); - else - addMatch(match,verbToObject); - } - if(++i % 100000==0) - LogInfo.logs("Number of lines: %s",i); - } - } - - private void log() { - - for(String verb: verbToObject.keySet()) { - Counter counter = verbToObject.get(verb); - List semClasses = Counters.toSortedList(counter, false); - for(String semClass: semClasses) { - LogInfo.logs("VerbSemClassExtractor v-->obj\tverb:%s\tsemclass:%s\tcount:%s",verb,semClass, - counter.getCount(semClass)); - } - } - for(String verb: verbToSubject.keySet()) { - Counter counter = verbToSubject.get(verb); - List semClasses = Counters.toSortedList(counter, false); - for(String semClass: semClasses) { - LogInfo.logs("VerbSemClassExtractor v-->sub\tverb:%s\tsemclass:%s\tcount:%s",verb,semClass, - counter.getCount(semClass)); - } - } - } - - private void addMatch(VerbSemClassMatch match, - Map> verbCounter) { - - MapUtils.putIfAbsent(verbCounter, match.verb, new ClassicCounter()); - Counter counter = verbCounter.get(match.verb); - counter.incrementCount(match.semClass); - } - /** - * Generates a map from words (verbs/semclass) to a set of co-occurring words with counts - * @param inFile - * @return - */ - public static Map> getWordToWordCounterMap() { - - LogInfo.begin_track_printAll("uploading verb to semcalss"); - Map> res = new HashMap>(); - - for(String line: IOUtils.readLines(opts.verbSemclassFile)) { - - String[] tokens = line.split("\t"); - String word1 = tokens[1].split(":")[1].split("\\s+")[0]; //get just the first word of the verb without preposition - String word2 = tokens[2].split(":")[1]; - double count = Double.parseDouble(tokens[3].split(":")[1]); - if(count>=3.0) { - MapUtils.putIfAbsent(res, word1, new ClassicCounter()); - res.get(word1).incrementCount(word2, count); - MapUtils.putIfAbsent(res, word2, new ClassicCounter()); - res.get(word2).incrementCount(word1, count); - } - } - LogInfo.logs("Uploaded counters for %s words", res.size()); - LogInfo.end_track(); - return res; - } - -} diff --git a/src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassMatcher.java b/src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassMatcher.java deleted file mode 100644 index ec1a3bd..0000000 --- a/src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassMatcher.java +++ /dev/null @@ -1,97 +0,0 @@ -package edu.stanford.nlp.sempre.paraphrase.rules; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import com.google.common.base.Joiner; - -import edu.stanford.nlp.ling.CoreAnnotations; -import edu.stanford.nlp.ling.CoreAnnotations.LemmaAnnotation; -import edu.stanford.nlp.ling.tokensregex.TokenSequenceMatcher; -import edu.stanford.nlp.ling.tokensregex.TokenSequencePattern; -import edu.stanford.nlp.pipeline.Annotation; -import edu.stanford.nlp.pipeline.StanfordCoreNLP; -import edu.stanford.nlp.util.CoreMap; - -public class VerbSemClassMatcher { - - // TOKEN REGEX TO MATCH VERB AND SEMANTIC CLASS - //matching non-wh objects - public static final TokenSequencePattern p1 = TokenSequencePattern.compile - ( - "[ { word:\"what\"} ] ([ { word:/.*/ } ]* ) [ { lemma:\"be\" } | {lemma:\"do\"} ] [ { word:/.*/ } ]* " + - "[ { tag:/NNP.*/ } ]+ ([ { tag:/VB.*/} ] ) [ { word:/.*/ } ]* "); - //matching wh objects - public static final TokenSequencePattern p2 = TokenSequencePattern.compile - ( - "( [ { word:/when|where|who/} ]) [ { word:/.*/ } ]* [ { lemma:\"be\" } | {lemma:\"do\"} ] [ { word:/.*/ } ]* [ { tag:/NNP.*/ } ]+ " + - "([ { tag:/VB.*/} ] ) [ { word:/.*/ } ]* "); - //matching non-wh subjects - public static final TokenSequencePattern p3 = TokenSequencePattern.compile - ( - "[ { word:\"what\"} ] ([ { tag:\"NN\" } | { tag:\"NNS\" }] ) " + - "([ { tag:/VB.*/} & !{lemma:/be|do|have/}] ) [ { word:/.*/ } ]* [ { tag:/NNP.*/ } ]+ [ { word:/.*/ } ]* "); - //matching wh subjects - public static final TokenSequencePattern p4 = TokenSequencePattern.compile - ( - "([ { word:\"who\"} ]) ([ { tag:/VB.*/} & !{lemma:/be|do|have/} ] ) [ { word:/.*/ } ]* [ { tag:/NNP.*/ } ]+ [ { word:/.*/ } ]* "); - - private Properties props = new Properties(); - private StanfordCoreNLP pipeline; - private static VerbSemClassMatcher extractor = null; - - public static VerbSemClassMatcher getInstance() { - if(extractor==null) extractor = new VerbSemClassMatcher(); - return extractor; - } - - private VerbSemClassMatcher() { - props.put("annotators", "tokenize, ssplit, pos, lemma"); - props.put("pos.model", "edu/stanford/nlp/models/pos-tagger/english-caseless-left3words-distsim.tagger"); //caseless model - pipeline = new StanfordCoreNLP(props); - } - - public VerbSemClassMatch match(String sentence) { - - Annotation annotation = new Annotation(sentence.toLowerCase()); - pipeline.annotate(annotation); - TokenSequenceMatcher m1 = p1.getMatcher(annotation.get(CoreAnnotations.TokensAnnotation.class)); - if(m1.find()) - return new VerbSemClassMatch(extractMatch(m1.groupNodes(2)), extractMatch(m1.groupNodes(1)), false); - TokenSequenceMatcher m2 = p2.getMatcher(annotation.get(CoreAnnotations.TokensAnnotation.class)); - if(m2.find()) - return new VerbSemClassMatch(extractMatch(m2.groupNodes(2)), extractMatch(m2.groupNodes(1)), false); - TokenSequenceMatcher m3 = p3.getMatcher(annotation.get(CoreAnnotations.TokensAnnotation.class)); - if(m3.find()) - return new VerbSemClassMatch(extractMatch(m3.groupNodes(2)), extractMatch(m3.groupNodes(1)), true); - TokenSequenceMatcher m4 = p3.getMatcher(annotation.get(CoreAnnotations.TokensAnnotation.class)); - if(m4.find()) - return new VerbSemClassMatch(extractMatch(m4.groupNodes(2)), extractMatch(m4.groupNodes(1)), true); - return null; - } - - public String extractMatch(List maps) { - List res = new LinkedList(); - for(CoreMap map: maps) - res.add(map.get(LemmaAnnotation.class)); - return Joiner.on(' ').join(res); - } - - public static class VerbSemClassMatch { - - public final String verb; - public final String semClass; - public final boolean isSubj; - - public VerbSemClassMatch(String verb, String semClass, boolean isSubj) { - this.verb = verb; - this.semClass = semClass; - this.isSubj = isSubj; - } - - public String toString() { - return verb+"\t"+semClass+"\t"+isSubj; - } - } -} diff --git a/src/edu/stanford/nlp/sempre/test/DerivationStreamTest.java b/src/edu/stanford/nlp/sempre/test/DerivationStreamTest.java new file mode 100644 index 0000000..c04e932 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/test/DerivationStreamTest.java @@ -0,0 +1,32 @@ +package edu.stanford.nlp.sempre.test; + +import edu.stanford.nlp.sempre.Derivation; +import edu.stanford.nlp.sempre.SingleDerivationStream; +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertEquals; + +/** + * @author Percy Liang + */ +public class DerivationStreamTest { + @Test + public void single() { + SingleDerivationStream s = new SingleDerivationStream() { + public Derivation createDerivation() { + return new Derivation.Builder().cat("NP").createDerivation(); + } + }; + assertEquals(true, s.hasNext()); + assertEquals(true, s.hasNext()); + assertEquals(true, s.hasNext()); + assertEquals("NP", s.next().cat); + assertEquals(false, s.hasNext()); + assertEquals(false, s.hasNext()); + + s = new SingleDerivationStream() { + public Derivation createDerivation() { return null; } + }; + assertEquals(false, s.hasNext()); + } +} diff --git a/src/edu/stanford/nlp/sempre/test/GrammarTest.java b/src/edu/stanford/nlp/sempre/test/GrammarTest.java new file mode 100644 index 0000000..349c6a3 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/test/GrammarTest.java @@ -0,0 +1,53 @@ +package edu.stanford.nlp.sempre.test; + +import edu.stanford.nlp.sempre.Grammar; +import edu.stanford.nlp.sempre.Rule; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.List; + +/** + * Test that the grammar correctly parsers rules. + */ +public class GrammarTest { + + public static Grammar makeTernaryGrammar() { + Grammar g = new Grammar(); + g.addStatement("(rule $ROOT ($X) (IdentityFn))"); + g.addStatement("(rule $X ($A $B $C) (IdentityFn))"); + g.addStatement("(rule $A (a) (ConstantFn (string a)))"); + g.addStatement("(rule $B (b) (ConstantFn (string b)))"); + g.addStatement("(rule $C (c) (ConstantFn (string c)))"); + return g; + } + + /** + * Checks that each rule is one of the following: + * $Cat => token + * $Cat => $Cat + * $Cat => token token + * $Cat => token $Cat + * $Cat => $Cat token + * $Cat => $Cat $Cat + */ + public boolean isValidBinaryGrammar(Grammar g) { + for (Rule rule : g.getRules()) { + if (!Rule.isCat(rule.lhs)) return false; + if (rule.rhs.size() != 1 && rule.rhs.size() != 2) return false; + } + + return true; + } + + @Test + public void testBinarizationOfTernaryGrammar() { + Grammar g = makeTernaryGrammar(); + List rules = g.getRules(); + Assert.assertEquals(6, rules.size()); + Assert.assertTrue(isValidBinaryGrammar(g)); + } + + + +} diff --git a/src/edu/stanford/nlp/sempre/test/GrammarValidityTest.java b/src/edu/stanford/nlp/sempre/test/GrammarValidityTest.java new file mode 100644 index 0000000..66399b1 --- /dev/null +++ b/src/edu/stanford/nlp/sempre/test/GrammarValidityTest.java @@ -0,0 +1,57 @@ +package edu.stanford.nlp.sempre.test; + +import edu.stanford.nlp.sempre.*; + +import org.testng.annotations.Test; + +import fig.basic.LogInfo; + +import java.util.*; +import java.nio.file.*; + +import static org.testng.AssertJUnit.assertEquals; + +/** + * Attempt to load all grammars to test for validity. + * + * @author Yushi Wang + */ + +public class GrammarValidityTest { + private String[] dataPaths = new String[] {"data/", "freebase/", "tables/", "regex/"}; + + @Test(groups = {"grammar"}) + public void readGrammars() { + try { + List successes = new ArrayList<>(), failures = new ArrayList<>(); + for (String dataPath : dataPaths) { + Files.walk(Paths.get(dataPath)).forEach(filePath -> { + try { + if (filePath.toString().toLowerCase().endsWith(".grammar")) { + Grammar test = new Grammar(); + LogInfo.logs("Reading grammar file: %s", filePath.toString()); + test.read(filePath.toString()); + LogInfo.logs("Finished reading", filePath.toString()); + successes.add(filePath.toString()); + } + } + catch (Exception ex) { + failures.add(filePath.toString()); + } + }); + } + LogInfo.begin_track("Following grammar tests passed:"); + for (String path : successes) + LogInfo.logs("%s", path); + LogInfo.end_track(); + LogInfo.begin_track("Following grammar tests failed:"); + for (String path : failures) + LogInfo.logs("%s", path); + LogInfo.end_track(); + assertEquals(0, failures.size()); + } + catch (Exception ex) { + LogInfo.logs(ex.toString()); + } + } +} diff --git a/src/edu/stanford/nlp/sempre/test/JavaExecutorTest.java b/src/edu/stanford/nlp/sempre/test/JavaExecutorTest.java index 973f9d3..e12a13e 100644 --- a/src/edu/stanford/nlp/sempre/test/JavaExecutorTest.java +++ b/src/edu/stanford/nlp/sempre/test/JavaExecutorTest.java @@ -15,23 +15,29 @@ public class JavaExecutorTest { private static Formula F(String s) { return Formula.fromString(s); } private static Value V(double x) { return new NumberValue(x); } - private static Value V(String x) { return new StringValue(x); } + private static Value V(String x) { return Values.fromString(x); } @Test public void numbers() { - assertEquals(V(8), executor.execute(F("(call + (number 3) (number 5))")).value); - assertEquals(V(6), executor.execute(F("(call + (call - (number 10) (number 9)) (number 5))")).value); - assertEquals(V(1), executor.execute(F("(call java.lang.Math.cos (number 0))")).value); + assertEquals(V(8), executor.execute(F("(call + (number 3) (number 5))"), null).value); + assertEquals(V(6), executor.execute(F("(call + (call - (number 10) (number 9)) (number 5))"), null).value); + assertEquals(V(1), executor.execute(F("(call java.lang.Math.cos (number 0))"), null).value); - assertEquals(V(1), executor.execute(F("((lambda x (call java.lang.Math.cos (var x))) (number 0))")).value); // Make sure beta reduction is called + assertEquals(V(1), executor.execute(F("((lambda x (call java.lang.Math.cos (var x))) (number 0))"), null).value); // Make sure beta reduction is called } @Test public void conditionals() { - assertEquals(V("no"), executor.execute(F("(call if (boolean false) (string yes) (string no))")).value); - assertEquals(V("yes"), executor.execute(F("(call if (call < (number 3) (number 4)) (string yes) (string no))")).value); + assertEquals(V("(string no)"), executor.execute(F("(call if (boolean false) (string yes) (string no))"), null).value); + assertEquals(V("(string yes)"), executor.execute(F("(call if (call < (number 3) (number 4)) (string yes) (string no))"), null).value); } @Test public void strings() { - assertEquals(V(5), executor.execute(F("(call .length (string hello))")).value); - assertEquals(V("abcdef"), executor.execute(F("(call .concat (string abc) (string def))")).value); + assertEquals(V(5), executor.execute(F("(call .length (string hello))"), null).value); + assertEquals(V("(string abcdef)"), executor.execute(F("(call .concat (string abc) (string def))"), null).value); + } + + @Test public void higherOrder() { + assertEquals(V("(list (number 10) (number 40))"), executor.execute(F("(call map (list (number 1) (number 4)) (lambda x (call * (number 10) (var x))))"), null).value); + assertEquals(V("(list (number 4))"), executor.execute(F("(call select (list (number 1) (number 4)) (lambda x (call == (number 0) (call % (var x) (number 2)))))"), null).value); + // assertEquals(V("(list (number 5))"), executor.execute(F("(call reduce (list (number 1) (number 4)) +)")).value); // Not implemented yet } } diff --git a/src/edu/stanford/nlp/sempre/test/JsonTest.java b/src/edu/stanford/nlp/sempre/test/JsonTest.java index 6a6b600..a30e307 100644 --- a/src/edu/stanford/nlp/sempre/test/JsonTest.java +++ b/src/edu/stanford/nlp/sempre/test/JsonTest.java @@ -4,8 +4,6 @@ import edu.stanford.nlp.sempre.*; import fig.basic.LogInfo; import org.testng.annotations.Test; -import java.util.ArrayList; - /** * Test JSON serialization and deserialization. */ @@ -14,10 +12,6 @@ public class JsonTest { return Json.writeValueAsStringHard(o); } - public static String S(Object o, Class view) { - return Json.writeValueAsStringHard(o, view); - } - public static T D(String s, Class klass) { return Json.readValueHard(s, klass); } @@ -29,8 +23,8 @@ public class JsonTest { public static boolean exampleEquals(Example a, Example b) { if (!a.id.equals(b.id)) return false; if (!a.utterance.equals(b.utterance)) return false; - if (a.derivConstraint != b.derivConstraint && - !a.derivConstraint.getFormulaPattern().equals(b.derivConstraint.getFormulaPattern())) + if (a.context != b.context && + !a.context.equals(b.context)) return false; if (a.targetFormula != b.targetFormula && !a.targetFormula.toString().equals(b.targetFormula.toString())) @@ -38,36 +32,18 @@ public class JsonTest { if (a.targetValue != b.targetValue && !a.targetValue.equals(b.targetValue)) return false; - if (a.getPredDerivations() != b.getPredDerivations() && - !a.getPredDerivations().equals(b.getPredDerivations())) - return false; - if (a.languageInfo != b.languageInfo && - !a.languageInfo.equals(b.languageInfo)) - return false; return true; } - @Test - public void testDerivation() { - Derivation.Builder b = new Derivation.Builder() - .cat("_cat") - .start(0).end(9) - .rule(Rule.nullRule) - .children(new ArrayList()) - .withStringFormulaFrom("_stringFormula"); - Derivation d = b.createDerivation(); - assert S(d).equals(S(b.createDerivation())); - - String there = S(d); - Derivation back = D(S(d), Derivation.class); - String thereAgain = S(back); - - assert there.equals(thereAgain); - assert d.equals(back); + public static Parser makeSimpleBeamParser() { + return new BeamParser(ParserTest.ABCTest().getParserSpec()); } @Test public void testExample() { + Builder builder = new Builder(); + builder.build(); + Example.Builder b = new Example.Builder(); Example ex = b .setId("id") @@ -77,34 +53,21 @@ public class JsonTest { LogInfo.log(S(ex)); assert exampleEquals(ex, D(S(ex), Example.class)); - ex.preprocess(); + ex.preprocess(new SimpleAnalyzer()); LogInfo.log(S(ex)); assert ex.languageInfo != null; assert !ex.languageInfo.tokens.isEmpty(); assert exampleEquals(ex, D(S(ex), Example.class)); ex = TestUtils.makeSimpleExample("1 2 3"); - - boolean tmpUseAnnotators = LanguageInfo.opts.useAnnotators; - LanguageInfo.opts.useAnnotators = false; - - ex.preprocess(); - LogInfo.log(S(ex, Example.JsonViews.WithDerivations.class)); - ParserTest.makeSimpleBeamParser().parse( - new Params(), - ex); - String there = S(ex, Example.JsonViews.WithDerivations.class); - Example back = D( - there, Example.class, Example.JsonViews.WithDerivations.class); - String thereAgain = S(back, Example.JsonViews.WithDerivations.class); + ex.preprocess(new SimpleAnalyzer()); + makeSimpleBeamParser().parse(new Params(), ex, true); + String there = S(ex); + Example back = D(there, Example.class); + String thereAgain = S(back); LogInfo.log(there); LogInfo.log(thereAgain); - - assert ex.getPredDerivations() != null; - assert !ex.getPredDerivations().isEmpty(); assert there.equals(thereAgain); assert exampleEquals(ex, back); - - LanguageInfo.opts.useAnnotators = tmpUseAnnotators; } } diff --git a/src/edu/stanford/nlp/sempre/test/LexiconTest.java b/src/edu/stanford/nlp/sempre/test/LexiconTest.java deleted file mode 100644 index 839a6c1..0000000 --- a/src/edu/stanford/nlp/sempre/test/LexiconTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package edu.stanford.nlp.sempre.test; - -import edu.stanford.nlp.sempre.fbalignment.lexicons.BinaryLexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.EntrySource; -import edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.BinaryLexicalEntry; -import edu.stanford.nlp.sempre.fbalignment.lexicons.LexicalEntry.UnaryLexicalEntry; -import edu.stanford.nlp.sempre.fbalignment.lexicons.UnaryLexicon; -import fig.basic.LogInfo; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.List; - -import static org.testng.AssertJUnit.assertEquals; - -public class LexiconTest { - - @Test(groups = "emnlp2013") - public void unary() throws IOException { - UnaryLexicon.opts.unaryLexiconFilePath = - "lib/fb_data/6/unaryInfoStringAndAlignment.txt"; - UnaryLexicon unary = new UnaryLexicon(); - boolean existsAlignment = false, existsStringMatch=false; - double popularity=0.0; - double intersection=0.0; - - List entries = unary.lookupEntries("continent"); - LogInfo.logs("Num of entries: %s",entries.size()); - for(UnaryLexicalEntry entry: entries) { - if(entry.formula.toString().equals("(fb:type.object.type fb:location.continent)")) { - if(entry.source==EntrySource.ALIGNMENT) { - existsAlignment=true; - intersection = entry.alignmentScores.get(UnaryLexicon.INTERSECTION); - } - else if(entry.source==EntrySource.STRING_MATCH) { - existsStringMatch=true; - popularity = entry.popularity; - } - } - } - assertEquals(true,existsAlignment); - assertEquals(true,existsStringMatch); - assertEquals(7.0, popularity, 0.0001); - assertEquals(5.0, intersection, 0.0001); - - existsAlignment = false; - existsStringMatch=false; - popularity=0.0; - intersection=0.0; - entries = unary.lookupEntries("lawyer"); - for(UnaryLexicalEntry entry: entries) { - if(entry.formula.toString().equals("(fb:people.person.profession fb:en.attorney)")) { - if(entry.source==EntrySource.ALIGNMENT) { - existsAlignment=true; - intersection = entry.alignmentScores.get(UnaryLexicon.INTERSECTION); - } - else if(entry.source==EntrySource.STRING_MATCH) { - existsStringMatch=true; - popularity = entry.popularity; - } - } - } - assertEquals(true,existsAlignment); - assertEquals(true,existsStringMatch); - assertEquals(12282.0, popularity, 0.0001); // Based on 93.exec (full Freebase) - assertEquals(26.0, intersection, 0.0001); - } - - @Test(groups = "emnlp2013") - public void binary() throws IOException { - BinaryLexicon.opts.keyToSortBy = BinaryLexicon.INTERSECTION; - - BinaryLexicon lexicon = new BinaryLexicon(); - List entries = lexicon.lookupEntries("bear in"); - BinaryLexicalEntry top = entries.get(0); - assertEquals("people born here", top.fbDescriptions.iterator().next()); - assertEquals("!fb:location.location.people_born_here", top.formula.toString()); - assertEquals("ALIGNMENT", top.source.toString()); - assertEquals(759773.0,top.popularity,0.00001); // Based on 93.exec (full Freebase) - assertEquals("fb:people.person",top.expectedType1); - assertEquals("fb:location.location",top.expectedType2); - assertEquals(16184.0,top.alignmentScores.get(BinaryLexicon.FB_TYPED),0.0001); - assertEquals(13856.0,top.alignmentScores.get(BinaryLexicon.INTERSECTION),0.0001); - assertEquals(15765.0, top.alignmentScores.get(BinaryLexicon.NL_TYPED),0.0001); - } -} diff --git a/src/edu/stanford/nlp/sempre/test/ParserTest.java b/src/edu/stanford/nlp/sempre/test/ParserTest.java index 5008ef9..8dc7884 100644 --- a/src/edu/stanford/nlp/sempre/test/ParserTest.java +++ b/src/edu/stanford/nlp/sempre/test/ParserTest.java @@ -1,8 +1,12 @@ package edu.stanford.nlp.sempre.test; import edu.stanford.nlp.sempre.*; +import fig.basic.LogInfo; import org.testng.annotations.Test; +import java.util.HashMap; +import java.util.Map; + import static org.testng.AssertJUnit.assertEquals; /** @@ -12,33 +16,142 @@ import static org.testng.AssertJUnit.assertEquals; * @author Percy Liang */ public class ParserTest { - public static BeamParser makeSimpleBeamParser() { - Executor executor = new FormulaMatchExecutor(); - FeatureExtractor extractor = new FeatureExtractor(executor); - FeatureExtractor.opts.featureDomains.add("rule"); - return new BeamParser(TestUtils.makeArithmeticGrammar(), extractor, executor); + // Collects a grammar, and some input/output test pairs + public abstract static class ParseTest { + public Grammar grammar; + ParseTest(Grammar g) { + this.grammar = g; + } + + public Parser.Spec getParserSpec() { + Executor executor = new JavaExecutor(); + FeatureExtractor extractor = new FeatureExtractor(executor); + FeatureExtractor.opts.featureDomains.add("rule"); + ValueEvaluator valueEvaluator = new ExactValueEvaluator(); + return new Parser.Spec(grammar, extractor, executor, valueEvaluator); + } + + public abstract void test(Parser parser); } - private static void checkNumDerivations(Parser parser, String utterance, int numExpected) { - Params params = new Params(); - Example ex = TestUtils.makeSimpleExample(utterance); - parser.parse(params, ex); + private static void checkNumDerivations(Parser parser, Params params, String utterance, String targetValue, int numExpected) { + Parser.opts.verbose = 1; + Example ex = TestUtils.makeSimpleExample(utterance, targetValue != null ? Value.fromString(targetValue) : null); + ParserState state = parser.parse(params, ex, targetValue != null); + + // Debug information + for (Derivation deriv : state.predDerivations) { + LogInfo.dbg(deriv.getAllFeatureVector()); + LogInfo.dbg(params.getWeights()); + LogInfo.dbgs("Score %f", deriv.computeScore(params)); + } + // parser.extractor.extractLocal(); assertEquals(numExpected, ex.getPredDerivations().size()); + if (numExpected > 0 && targetValue != null) + assertEquals(targetValue, ex.getPredDerivations().get(0).value.toString()); + } + private static void checkNumDerivations(Parser parser, String utterance, String targetValue, int numExpected) { + checkNumDerivations(parser, new Params(), utterance, targetValue, numExpected); } - private static void checkNumDerivations(Parser parser) { - checkNumDerivations(parser, "1 +", 0); - checkNumDerivations(parser, "1", 1); - checkNumDerivations(parser, "1 2", 1); - checkNumDerivations(parser, "1 2 3", 2); - checkNumDerivations(parser, "10 20 30 40 50 60", 42); + static ParseTest ABCTest() { + return new ParseTest(TestUtils.makeAbcGrammar()) { + @Override + public void test(Parser parser) { + checkNumDerivations(parser, "a +", null, 0); + checkNumDerivations(parser, "a", "(string a)", 1); + checkNumDerivations(parser, "a b", "(string a,b)", 1); + checkNumDerivations(parser, "a b c", "(string a,b,c)", 2); + checkNumDerivations(parser, "a b c a b c", "(string a,b,c,a,b,c)", 42); + } + }; } - @Test - public void checkNumDerivations() { - LanguageInfo.opts.useAnnotators = false; - checkNumDerivations(makeSimpleBeamParser()); - LanguageInfo.opts.useAnnotators = true; - // Include other parsers here... + static ParseTest ArithmeticTest() { + return new ParseTest(TestUtils.makeArithmeticGrammar()) { + @Override + public void test(Parser parser) { + checkNumDerivations(parser, "1 + ", null, 0); + checkNumDerivations(parser, "1 plus 2", "(number 3)", 1); + checkNumDerivations(parser, "2 times 3", "(number 6)", 1); + checkNumDerivations(parser, "1 plus times 3", null, 0); + checkNumDerivations(parser, "times", null, 0); + } + }; + }; + + // Create parsers + @Test public void checkBeamNumDerivationsForABCGrammar() { + Parser.opts.coarsePrune = false; + ParseTest p; + p = ABCTest(); + p.test(new BeamParser(p.getParserSpec())); + p = ArithmeticTest(); + p.test(new BeamParser(p.getParserSpec())); } + @Test public void checkCoarseBeamNumDerivations() { + Parser.opts.coarsePrune = true; + ParseTest p; + p = ABCTest(); + p.test(new BeamParser(p.getParserSpec())); + p = ArithmeticTest(); + p.test(new BeamParser(p.getParserSpec())); + } + + @Test(groups = "reinforcement") public void checkReinforcementNumDerivations() { + ParseTest p; + p = ABCTest(); + p.test(new ReinforcementParser(p.getParserSpec())); + p = ArithmeticTest(); + p.test(new ReinforcementParser(p.getParserSpec())); + // TODO(chaganty): test more thoroughly + } + + @Test public void checkFloatingNumDerivations() { + // Make it behave like the BeamParser + FloatingParser.opts.defaultIsFloating = false; + ParseTest p; + p = ABCTest(); + p.test(new FloatingParser(p.getParserSpec())); + p = ArithmeticTest(); + p.test(new FloatingParser(p.getParserSpec())); + + // If floating, should get more hypotheses + FloatingParser.opts.defaultIsFloating = true; + Parser parser = new FloatingParser(ABCTest().getParserSpec()); + FloatingParser.opts.maxDepth = 2; + checkNumDerivations(parser, "ignore", null, 3); + FloatingParser.opts.maxDepth = 3; + checkNumDerivations(parser, "ignore", null, 3 + 3 * 3); + } + + // TODO(chaganty): verify that things are ranked appropriately + public void checkRankingArithmetic(Parser parser) { + Params params = new Params(); + Map features = new HashMap<>(); + features.put("rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))", 1.0); + features.put("rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))", -1.0); + params.update(features); + checkNumDerivations(parser, params, "2 and 3", "(number 5)", 2); + + params = new Params(); + features.put("rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call + (var x) (var y)))))", -1.0); + features.put("rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call * (var x) (var y)))))", 1.0); + params.update(features); + checkNumDerivations(parser, params, "2 and 3", "(number 6)", 2); + } + @Test void checkRankingSimple() { + checkRankingArithmetic(new BeamParser(ArithmeticTest().getParserSpec())); + } + @Test void checkRankingReinforcement() { + checkRankingArithmetic(new ReinforcementParser(ArithmeticTest().getParserSpec())); + } + @Test void checkRankingFloating() { + FloatingParser.opts.defaultIsFloating = false; + checkRankingArithmetic(new FloatingParser(ArithmeticTest().getParserSpec())); + } + + // TODO(chaganty): verify the parser gradients + + } diff --git a/src/edu/stanford/nlp/sempre/test/SemTypeTest.java b/src/edu/stanford/nlp/sempre/test/SemTypeTest.java index 288e6da..ccfd2e6 100644 --- a/src/edu/stanford/nlp/sempre/test/SemTypeTest.java +++ b/src/edu/stanford/nlp/sempre/test/SemTypeTest.java @@ -1,37 +1,66 @@ package edu.stanford.nlp.sempre.test; -import edu.stanford.nlp.sempre.SemType; -import fig.basic.LispTree; +import static org.testng.AssertJUnit.assertEquals; + import org.testng.annotations.Test; +import edu.stanford.nlp.sempre.*; +import fig.basic.*; + /** - * Test Formulas. + * Test type system. * @author Percy Liang + * @author ppasupat */ public class SemTypeTest { // For testing private static SemType T(String str) { return SemType.fromLispTree(LispTree.proto.parseFromString(str)); } - private static void verify(SemType predType, SemType wantedType) { - if (!predType.toString().equals(wantedType.toString())) - throw new RuntimeException(String.format("Wanted %s, but got %s", wantedType, predType)); + + private static void verifyEquals(SemType predType, SemType wantedType) { + assertEquals(wantedType.toString(), predType.toString()); } - private static void verifySupertypeOf(String supertype, String subtype) { - if (!T(supertype).isSupertypeOf(T(subtype))) - throw new RuntimeException(supertype + " is not a supertype of " + subtype); + + private static void verifyMeet(String t1, String t2) { verifyMeet(t1, t2, t2); } + private static void verifyMeet(String t1, String t2, String t) { + verifyEquals(T(t1).meet(T(t2)), T(t)); + verifyEquals(T(t2).meet(T(t1)), T(t)); } @Test public void simpleSemType() { - verify(T("city").meet(T("city")), T("city")); - verify(T("city").meet(T("country")), T("(union)")); - verify(T("city").meet(T("(union city country)")), T("city")); - verify(T("(union country city river)").meet(T("(union city country)")), T("(union country city)")); - verify(T("(-> city fb:type.int)").apply(T("(union city country)")), T("fb:type.int")); - verify(T("(-> city fb:type.int fb:type.float)").apply(T("(union city country)")).apply(T("fb:type.int")), T("fb:type.float")); - verify(T("fb:type.datetime").apply(T("fb:common.topic")), T("(union)")); - verify(T("(-> fb:location.citytown fb:type.datetime)").apply(T("fb:location.location")), T("(union)")); - verify(T("(-> fb:location.location fb:type.datetime)").apply(T("fb:location.citytown")), T("fb:type.datetime")); - verifySupertypeOf("(-> fb:location.location fb:type.number)", "(-> fb:location.location fb:type.float)"); + SemTypeHierarchy.opts.failOnUnknownTypes = false; + verifyMeet("city", "city"); + verifyMeet("city", "country", "(union)"); + verifyMeet("city", "(union city country)", "city"); + verifyMeet("(union city country river)", "(union city country)"); + + verifyEquals(T("(-> city fb:type.int)").apply(T("(union city country)")), T("fb:type.int")); + verifyEquals(T("(-> city fb:type.int fb:type.float)").apply(T("(union city country)")).apply(T("fb:type.int")), T("fb:type.float")); + verifyEquals(T("fb:type.datetime").apply(T("fb:common.topic")), T("(union)")); + verifyEquals(T("(-> fb:type.int fb:type.datetime)").apply(T("fb:type.number")), T("fb:type.datetime")); + verifyEquals(T("(-> fb:type.number fb:type.datetime)").apply(T("fb:type.int")), T("fb:type.datetime")); + + verifyMeet("(-> fb:location.location fb:type.number)", "(-> fb:location.location fb:type.float)"); + verifyMeet("fb:common.topic", "fb:location.location"); + verifyMeet("fb:type.any", "fb:type.boolean"); + verifyMeet("fb:type.any", "fb:type.number"); + verifyMeet("fb:type.any", "fb:type.datetime"); + verifyMeet("fb:type.any", "fb:type.cvt"); + verifyMeet("fb:type.any", "fb:type.text"); + verifyMeet("fb:type.any", "fb:location.location"); + verifyMeet("fb:type.any", "fb:common.topic"); + + verifyMeet("fb:common.topic", "fb:common.topic"); + verifyMeet("top", "(-> t t)"); + verifyMeet("top", "fb:type.datetime"); + verifyMeet("top", "(union a b)"); + + verifyMeet("(-> (-> a b) top)", "(-> top (-> a b))", "(-> (-> a b) (-> a b))"); + verifyMeet("(-> (union city country) person)", "(-> city (union person dog))", "(-> city person)"); + } + + public static void main(String[] args) { + new SemTypeTest().simpleSemType(); } } diff --git a/src/edu/stanford/nlp/sempre/test/SemanticFnTest.java b/src/edu/stanford/nlp/sempre/test/SemanticFnTest.java index dfb57ce..9342dd5 100644 --- a/src/edu/stanford/nlp/sempre/test/SemanticFnTest.java +++ b/src/edu/stanford/nlp/sempre/test/SemanticFnTest.java @@ -1,8 +1,10 @@ package edu.stanford.nlp.sempre.test; import edu.stanford.nlp.sempre.*; +import fig.basic.LispTree; import org.testng.annotations.Test; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -15,28 +17,109 @@ import static org.testng.AssertJUnit.assertEquals; public class SemanticFnTest { private static Formula F(String s) { return Formula.fromString(s); } - void check(Formula target, List derivations) { - if (derivations.size() != 1) throw new RuntimeException("Expected 1 derivation, got " + derivations); - assertEquals(target, derivations.get(0).formula); + void check(Formula target, DerivationStream derivations) { + if (!derivations.hasNext()) throw new RuntimeException("Expected 1 derivation, got " + derivations); + assertEquals(target, derivations.next().formula); + } + + void check(Formula target, String utterance, SemanticFn fn, List children) { + Example ex = TestUtils.makeSimpleExample(utterance); + check(target, fn.call(ex, new SemanticFn.CallInfo(null, 0, ex.numTokens(), Rule.nullRule, children))); } void check(Formula target, String utterance, SemanticFn fn) { - Example ex = TestUtils.makeSimpleExample(utterance); List empty = Collections.emptyList(); - check(target, fn.call(ex, new SemanticFn.CallInfo(null, 0, ex.numTokens(), Rule.nullRule, empty))); + check(target, utterance, fn, empty); + } + + void checkNumDerivations(DerivationStream derivations, int num) { + assertEquals(num, derivations.estimatedSize()); } @Test public void constantFn() { + LanguageAnalyzer.setSingleton(new SimpleAnalyzer()); check(F("(number 3)"), "whatever", new ConstantFn(F("(number 3)"))); } - @Test public void numberFn() { - check(F("(number 35000)"), "thirty-five thousand", new NumberFn()); + Derivation D(Formula f) { + return (new Derivation.Builder()) + .formula(f) + .prob(1.0) + .createDerivation(); } - @Test public void dateFn() { - check(F("(date 2013 8 7)"), "August 7, 2013", new DateFn()); - check(F("(date 1982 -1 -1)"), "1982", new DateFn()); - check(F("(date -1 6 4)"), "june 4", new DateFn()); + LispTree T(String str) { + return LispTree.proto.parseFromString(str); } + + // TODO(chaganty): Test bridge fn - requires freebase + + @Test public void concatFn() { + LanguageAnalyzer.setSingleton(new SimpleAnalyzer()); + check(F("(string \"a b\")"), "a b", new ConcatFn(" "), + Arrays.asList(D(F("(string a)")), D(F("(string b)")))); + } + + // TODO(chaganty): Test context fn + + @Test public void filterPosTagFn() { + LanguageAnalyzer.setSingleton(new SimpleAnalyzer()); + FilterPosTagFn filter = new FilterPosTagFn(); + filter.init(T("(FilterPosTagFn token NNP)")); + Derivation child = new Derivation.Builder().createDerivation(); + Example ex = TestUtils.makeSimpleExample("where is Obama"); + assertEquals(filter.call(ex, + new SemanticFn.CallInfo(null, 0, 1, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + assertEquals(filter.call(ex, + new SemanticFn.CallInfo(null, 1, 2, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + assertEquals(filter.call(ex, + new SemanticFn.CallInfo(null, 2, 3, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + } + + @Test public void filterSpanLengthFn() { + LanguageAnalyzer.setSingleton(new SimpleAnalyzer()); + FilterSpanLengthFn filter = new FilterSpanLengthFn(); + filter.init(T("(FilterSpanLengthFn 2)")); + Derivation child = new Derivation.Builder().createDerivation(); + Example ex = TestUtils.makeSimpleExample("This is a sentence with some words"); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 1, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 2, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 2, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + + filter = new FilterSpanLengthFn(); + filter.init(T("(FilterSpanLengthFn 2 4)")); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 1, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 2, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 3, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 4, Rule.nullRule, Collections.singletonList(child))).hasNext(), + true); + assertEquals( + filter.call(ex, new SemanticFn.CallInfo(null, 0, 5, Rule.nullRule, Collections.singletonList(child))).hasNext(), + false); + } + + // TODO(chaganty): Test fuzzy match fn + // TODO(chaganty): Test identity fn + // TODO(chaganty): Test join fn + // TODO(chaganty): Test lexicon fn + // TODO(chaganty): Test merge fn + // TODO(chaganty): Test select fn + // TODO(chaganty): Test simple lexicon fn + } diff --git a/src/edu/stanford/nlp/sempre/test/SystemSanityTest.java b/src/edu/stanford/nlp/sempre/test/SystemSanityTest.java index 83f1f76..6035405 100644 --- a/src/edu/stanford/nlp/sempre/test/SystemSanityTest.java +++ b/src/edu/stanford/nlp/sempre/test/SystemSanityTest.java @@ -2,8 +2,11 @@ package edu.stanford.nlp.sempre.test; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; + import edu.stanford.nlp.sempre.*; import fig.basic.Pair; +import fig.basic.Evaluation; + import org.testng.annotations.Test; import java.util.Collections; @@ -33,7 +36,7 @@ public class SystemSanityTest { private static Dataset makeDataset() { Dataset d = new Dataset(); d.readFromPathPairs(Collections.singletonList( - Pair.newPair("train", "data/unittest-learn.examples.json"))); + Pair.newPair("train", "freebase/data/unittest-learn.examples"))); return d; } @@ -43,13 +46,14 @@ public class SystemSanityTest { return evals; } - @Test + @Test(groups = { "sparql", "corenlp" }) public void easyEndToEnd() { + LanguageAnalyzer.setSingleton(new SimpleAnalyzer()); // Make sure learning works Dataset dataset = makeDataset(); String[] grammarPaths = new String[] { - "data/unittest-learn.grammar", - "data/unittest-learn-ccg.grammar", + "freebase/data/unittest-learn.grammar", + "freebase/data/unittest-learn-ccg.grammar", }; for (String grammarPath : grammarPaths) { Builder builder = makeBuilder(grammarPath); diff --git a/src/edu/stanford/nlp/sempre/test/TestUtils.java b/src/edu/stanford/nlp/sempre/test/TestUtils.java index 9e41a51..cb44050 100644 --- a/src/edu/stanford/nlp/sempre/test/TestUtils.java +++ b/src/edu/stanford/nlp/sempre/test/TestUtils.java @@ -5,12 +5,36 @@ import edu.stanford.nlp.sempre.*; /** * Useful utilities and dummy system components for writing tests. * - * TODO: it's hard to come up with generally good defaults. - * * @author Roy Frostig */ -public class TestUtils { +public final class TestUtils { + private TestUtils() { } + + public static Grammar makeAbcGrammar() { + Grammar g = new Grammar(); + g.addStatement("(rule $X (a) (ConstantFn (string a)))"); + g.addStatement("(rule $X (b) (ConstantFn (string b)))"); + g.addStatement("(rule $X (c) (ConstantFn (string c)))"); + g.addStatement("(rule $X ($X $X) (ConcatFn ,))"); + g.addStatement("(rule $ROOT ($X) (IdentityFn))"); + return g; + } + public static Grammar makeArithmeticGrammar() { + Grammar g = new Grammar(); + g.addStatement("(rule $Expr ($TOKEN) (NumberFn))"); + g.addStatement("(rule $Expr ($Expr $Partial) (JoinFn backward))"); + g.addStatement("(rule $Partial ($Operator $Expr) (JoinFn forward))"); + g.addStatement("(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))"); + g.addStatement("(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))"); + g.addStatement("(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))"); + g.addStatement("(rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))"); + g.addStatement("(rule $ROOT ($Expr) (IdentityFn))"); + return g; + } + + + public static Grammar makeNumberConcatGrammar() { Grammar g = new Grammar(); g.addStatement("(rule $Number ($TOKEN) (NumberFn))"); g.addStatement("(rule $Number ($Number $Number) (ConcatFn ,))"); @@ -20,7 +44,7 @@ public class TestUtils { public static Builder makeSimpleBuilder() { Builder builder = new Builder(); - builder.grammar = makeArithmeticGrammar(); + builder.grammar = makeNumberConcatGrammar(); builder.executor = new FormulaMatchExecutor(); builder.buildUnspecified(); return builder; @@ -42,12 +66,16 @@ public class TestUtils { return makeSimpleLearner(makeSimpleBuilder(), makeSimpleDataset()); } - public static Example makeSimpleExample(String utterance) { + public static Example makeSimpleExample(String utterance) { return makeSimpleExample(utterance, null); } + public static Example makeSimpleExample(String utterance, Value targetValue) { + Builder builder = new Builder(); + builder.build(); Example ex = new Example.Builder() .setId("_id") .setUtterance(utterance) + .setTargetValue(targetValue) .createExample(); - ex.preprocess(); + ex.preprocess(LanguageAnalyzer.getSingleton()); return ex; } } diff --git a/src/edu/stanford/nlp/sempre/test/TypeInferenceTest.java b/src/edu/stanford/nlp/sempre/test/TypeInferenceTest.java new file mode 100644 index 0000000..2de343f --- /dev/null +++ b/src/edu/stanford/nlp/sempre/test/TypeInferenceTest.java @@ -0,0 +1,117 @@ +package edu.stanford.nlp.sempre.test; + +import static org.testng.AssertJUnit.assertEquals; + +import org.testng.annotations.Test; + +import edu.stanford.nlp.sempre.*; + +/** + * Test type inference. + * @author Percy Liang + * @author ppasupat + */ +public class TypeInferenceTest { + // For testing + private static final CustomTypeLookup typeLookup = new CustomTypeLookup(); + private static Formula F(String str) { return Formula.fromString(str); } + private static SemType T(String str) { return SemType.fromString(str); } + private static SemType FT(String str) { return TypeInference.inferType(F(str), typeLookup); } + + static class CustomTypeLookup implements TypeLookup { + + + @Override + public SemType getEntityType(String entity) { + return null; + } + + @Override + public SemType getPropertyType(String property) { + switch (property) { + case "fb:location.location.area": + return T("(-> fb:type.number fb:location.location)"); + case "fb:people.person.date_of_birth": + return T("(-> fb:type.datetime fb:people.person)"); + case "fb:people.person.parents": + return T("(-> fb:people.person fb:people.person)"); + case "fb:people.person.place_of_birth": + return T("(-> fb:location.location fb:people.person)"); + case "fb:people.person.profession": + return T("(-> fb:people.profession fb:people.person)"); + default: + return null; + } + } + + } + + void check(String fstr, String tstr) { + System.out.println("check " + fstr + " " + tstr); + assertEquals(T(tstr).toString(), FT(fstr).toString()); + } + + @Test public void simpleSemType() { + check("(fb:location.location.area (>= (number 200)))", "fb:location.location"); + + check("(number 3)", "fb:type.number"); + check("(string foo)", "fb:type.text"); + check("(date 1981 1 1)", "fb:type.datetime"); + check("fb:en.barack_obama", "fb:common.topic"); // Don't have getEntityTypes + check("fb:people.person.place_of_birth", "(-> fb:location.location fb:people.person)"); + + // Join + check("(fb:type.object.type fb:location.location)", "fb:location.location"); + check("(fb:people.person.place_of_birth (fb:type.object.type fb:location.location))", "fb:people.person"); + check("(!fb:people.person.place_of_birth (fb:type.object.type fb:location.location))", "(union)"); + + // Merge + check("(and (fb:type.object.type fb:common.topic) (fb:people.person.place_of_birth fb:en.seattle))", "fb:people.person"); + check("(and (fb:type.object.type fb:location.location) (fb:people.person.place_of_birth fb:en.seattle))", "(union)"); + + // Mark + check("(mark x (fb:people.person.parents (var x)))", "fb:people.person"); + check("(mark x (fb:people.person.place_of_birth (var x)))", "(union)"); + + // Lambda + check("(lambda x (fb:people.person.place_of_birth (var x)))", "(-> fb:location.location fb:people.person)"); + check("(lambda x (!fb:people.person.place_of_birth (var x)))", "(-> fb:people.person fb:location.location)"); + check("(lambda x (fb:people.person.place_of_birth (var x)))", "(-> fb:location.location fb:people.person)"); + check("(lambda x (!fb:people.person.place_of_birth (var x)))", "(-> fb:people.person fb:location.location)"); + check("(lambda x (!fb:people.person.profession (fb:people.person.place_of_birth (var x))))", "(-> fb:location.location fb:people.profession)"); + check("(lambda b ((var b) (fb:type.object.type fb:people.person)))", "(-> (-> fb:people.person top) top)"); + // Note: and the other way doesn't work, since we don't propagate everything. + check("(lambda b (and (fb:type.object.type fb:location.location) ((var b) (fb:type.object.type fb:people.person))))", "(-> (-> fb:people.person fb:location.location) fb:location.location)"); + check("(lambda x (lambda y ((var x) (var y))))", "(-> (-> top top) (-> top top))"); + check("(lambda x (lambda x (fb:people.person.place_of_birth (var x))))", "(-> top (-> fb:location.location fb:people.person))"); // No variable capture + + // Aggregation + check("(lambda x (not (var x)))", "(-> fb:type.any fb:type.any)"); + check("(lambda x (count (var x)))", "(-> fb:type.any fb:type.number)"); + check("(lambda x (count (fb:people.person.place_of_birth (var x))))", "(-> fb:location.location fb:type.number)"); + + // Arithmetic + check("(+ (number 3) (number 4))", "fb:type.number"); + check("(+ (date 1981 1 1) (string 4))", "(union)"); + check("(- (date 1982 1 1) (date 1981 1 1))", "fb:type.datetime"); // Future: should be a different duration type + + // Reverse + check("(reverse fb:people.person.place_of_birth)", "(-> fb:people.person fb:location.location)"); + + // Superlative + check("(argmax 1 1 (fb:type.object.type fb:people.person) fb:people.person.date_of_birth)", "fb:people.person"); + check("(argmax 1 1 (fb:type.object.type fb:common.topic) fb:people.person.date_of_birth)", "fb:people.person"); + check("(argmax 1 1 (fb:type.object.type fb:common.topic) (reverse (lambda x (number 3))))", "fb:common.topic"); + check("(lambda x (lambda y (argmax 1 1 (var x) (var y))))", "(-> fb:type.any (-> (-> (union fb:type.number fb:type.datetime) fb:type.any) fb:type.any))"); + + // Call + check("(call Math.cos (number 0))", "fb:type.float"); + check("(call Math.cos (string abc))", "(union)"); + check("(lambda x (lambda y (call .concat (var x) (var y))))", "(-> fb:type.text (-> fb:type.text fb:type.text))"); + check("(lambda x (call .length (var x)))", "(-> fb:type.text fb:type.int)"); + } + + public static void main(String[] args) { + new TypeInferenceTest().simpleSemType(); + } +} diff --git a/src/edu/stanford/nlp/sempre/vis/BeamFigures.java b/src/edu/stanford/nlp/sempre/vis/BeamFigures.java deleted file mode 100644 index 6fccd13..0000000 --- a/src/edu/stanford/nlp/sempre/vis/BeamFigures.java +++ /dev/null @@ -1,226 +0,0 @@ -package edu.stanford.nlp.sempre.vis; - -import com.google.common.base.Joiner; -import com.google.common.primitives.Ints; -import edu.stanford.nlp.sempre.Derivation; -import edu.stanford.nlp.sempre.Example; -import edu.stanford.nlp.sempre.Vis; -import fig.basic.IOUtils; -import fig.basic.IntTriple; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.exec.Execution; - -import java.io.File; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** @author Roy Frostig */ -public class BeamFigures { - private final List execPaths; - - public BeamFigures(List execPaths) { - this.execPaths = execPaths; - } - - private void logFileList(List files) { - LogInfo.begin_track("Files"); - for (File file : files) - LogInfo.logs("%s", file); - LogInfo.end_track(); - } - - /** @param files Examples file from each iteration in turn. */ - private void writeBeamDistributions(List files, String group) { - String basePath = "beamfigs-dists-" + group + ".vis"; - - LogInfo.logs("Example files: %s", files); - - List countsPerIter = new ArrayList(); - int iter = 0, e = 0, beamSize = 0; - for (File file : files) { - LogInfo.logs("Counting histogram %d.%s", iter, group); - e = 0; - int[] counts = new int[0]; - for (Example example : Vis.getExamples(file)) { - final List beam = example.getPredDerivations(); - beamSize = Math.max(beamSize, beam.size()); - if (beamSize > counts.length) - counts = Utils.resize(counts, beamSize); - for (int i = 0; i < beam.size(); i++) - if (beam.get(i).getCompatibility() == 1.0d) - counts[i]++; - e++; - } - countsPerIter.add(counts); - iter++; - } - - LogInfo.log("Writing " + basePath); - String outPath = Execution.getFile(basePath); - PrintWriter out = IOUtils.openOutHard(outPath); - - out.printf("examples %d\n", e); - out.printf("iters %d\n", iter); - out.printf("beamsize %d\n", beamSize); - for (int[] counts : countsPerIter) - out.println(Joiner.on(' ').join(Ints.asList(counts))); - - LogInfo.log("Done"); - - out.flush(); - out.close(); - } - - /** @param files Examples file from each iteration in turn. */ - private void writeExampleMetaAndBeamItemsAndMaps(List files, - String group) { - String basePathMeta = "beamfigs-meta-" + group + ".vis"; - String basePathCorBmp = "beamfigs-corbmp-" + group + ".vis"; - String basePathItems = "beamfigs-items-" + group + ".vis"; - - LogInfo.logs("Example files: %s", files); - - List correctPoints = new ArrayList(); - List exampleMetadata = new ArrayList(); - - int iter = 0, e = 0, beamSize = 0; - for (File file : files) { - LogInfo.logs("Processing %d.%s", iter, group); - e = 0; - for (Example example : Vis.getExamples(file)) { - final List beam = example.getPredDerivations(); - beamSize = Math.max(beamSize, beam.size()); - for (int i = 0; i < beam.size(); i++) - if (beam.get(i).getCompatibility() == 1.0d) - correctPoints.add(new IntTriple(iter, e, i)); - if (iter == 0) { - exampleMetadata.add( - String.format( - "%d %d %s", - beam.size(), - example.getTokens().size(), - Joiner.on(' ').join(example.getTokens()))); - } else { - exampleMetadata.add("" + beam.size()); - } - e++; - } - iter++; - } - - LogInfo.log("Writing " + basePathCorBmp); - String outPath = Execution.getFile(basePathCorBmp); - PrintWriter out = IOUtils.openOutHard(outPath); - - out.printf("examples %d\n", e); - out.printf("iters %d\n", iter); - out.printf("beamsize %d\n", beamSize); - for (IntTriple pt : correctPoints) - out.printf("%d %d %d\n", pt.first, pt.second, pt.third); - - out.flush(); - out.close(); - LogInfo.log("Done"); - - LogInfo.log("Writing " + basePathMeta); - outPath = Execution.getFile(basePathMeta); - out = IOUtils.openOutHard(outPath); - - out.printf("examples %d\n", e); - out.printf("iters %d\n", iter); - out.printf("beamsize %d\n", beamSize); - - for (String s : exampleMetadata) - out.println(s); - - out.flush(); - out.close(); - LogInfo.log("Done"); - - LogInfo.log("Writing " + basePathItems); - outPath = Execution.getFile(basePathItems); - out = IOUtils.openOutHard(outPath); - - out.printf("examples %d\n", e); - out.printf("iters %d\n", iter); - out.printf("beamsize %d\n", beamSize); - - for (File file : files) { - LogInfo.logs("Processing examples from %s", file); - for (Example example : Vis.getExamples(file)) { - final List beam = example.getPredDerivations(); - for (Derivation deriv : beam) - out.printf( - "%.5f %.5f %.5f\n", - deriv.getCompatibility(), - deriv.getScore(), - deriv.getProb()); - } - } - - out.flush(); - out.close(); - LogInfo.log("Done"); - } - - /** @param files Example file from each iteration in turn. */ - private void writeBeamDeltas(List files, String group) { - String basePath = "beamfigs-deltas-" + group + ".vis"; - - LogInfo.begin_track("Collecting deltas for all examples"); - LogInfo.logs("Example files: %s", files); - - List>> deltasPerIterPerExample = new ArrayList>>(); - int e = 0, iter = 0, beamSize = 0; - for (List row : Vis.zipExamples(files)) { - LogInfo.log("example " + e); - final Example targetExample = row.get(row.size() - 1); - final Map targetBeamPositions = Utils.indicesOf(targetExample.getPredDerivations()); - beamSize = targetBeamPositions.size(); - - List> deltasPerIter = new ArrayList>(); - for (iter = 0; iter < row.size(); iter++) { - List deltas = new ArrayList(); - List beam = row.get(iter).getPredDerivations(); - for (int i = 0; i < beam.size(); i++) { - int targetPos = MapUtils.get(targetBeamPositions, beam.get(i), targetBeamPositions.size()); - deltas.add(i - targetPos); - } - deltasPerIter.add(deltas); - } - deltasPerIterPerExample.add(deltasPerIter); - e++; - } - - String outPath = Execution.getFile(basePath); - PrintWriter out = IOUtils.openOutHard(outPath); - LogInfo.log("Writing " + basePath); - - out.printf("examples %d\n", e); - out.printf("iters %d\n", iter); - out.printf("beamsize %d\n", beamSize); - for (List> deltasPerIter : deltasPerIterPerExample) - for (List deltas : deltasPerIter) - out.println(Joiner.on(' ').join(deltas)); - - LogInfo.log("Done"); - out.flush(); - out.close(); - } - - public void write(String execPath) { - for (String group : new String[]{"train", "dev"}) { - writeBeamDistributions(Vis.getExecIterFiles(execPath, group), group); - writeExampleMetaAndBeamItemsAndMaps(Vis.getExecIterFiles(execPath, group), group); - writeBeamDeltas(Vis.getExecIterFiles(execPath, group), group); - } - } - - public void writeAll() { - for (String execPath : execPaths) - write(execPath); - } -} diff --git a/src/edu/stanford/nlp/sempre/vis/ConfusionMatrices.java b/src/edu/stanford/nlp/sempre/vis/ConfusionMatrices.java deleted file mode 100644 index 1b25266..0000000 --- a/src/edu/stanford/nlp/sempre/vis/ConfusionMatrices.java +++ /dev/null @@ -1,127 +0,0 @@ -package edu.stanford.nlp.sempre.vis; - -import com.google.common.base.Joiner; -import edu.stanford.nlp.sempre.Derivation; -import edu.stanford.nlp.sempre.Example; -import edu.stanford.nlp.sempre.Vis; -import fig.basic.LogInfo; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -/** - * Visualize confusion matrices for binary-classifier-based derivation rankings - * in the semantic parser. - * - * @author Roy Frostig - */ -public class ConfusionMatrices { - - private final List execPaths; - - public ConfusionMatrices(List execPaths) { - this.execPaths = execPaths; - } - - public boolean logs(int iter, String group) { - List files = Vis.getFilesPerExec(execPaths, iter, group); - - if (files == null) - return false; - - LogInfo.logs("Reading files: %s", files); - final int n = files.size(); - - List softMs = new ArrayList(n); - List hardMs = new ArrayList(n); - for (int i = 0; i < n; i++) { - softMs.add(new ConfusionMatrix()); - hardMs.add(new ConfusionMatrix()); - } - - final double ct = 0.5d; - final double pt = 0.5d; - - for (List row : Vis.zipExamples(files)) { - for (int i = 0; i < n; i++) { - Example ex = row.get(i); - ConfusionMatrix softM = softMs.get(i); - ConfusionMatrix hardM = hardMs.get(i); - updateConfusionMatrix(softMs.get(i), ex, -1.0d, -1.0d); - updateConfusionMatrix(hardMs.get(i), ex, ct, pt); - } - } - - LogInfo.begin_track("Soft"); - logsMatrices(softMs); - LogInfo.end_track(); - - LogInfo.begin_track("Hard (compatThresh=%.2f, probThresh=%.2f)", ct, pt); - logsMatrices(hardMs); - LogInfo.end_track(); - - return true; - } - - private static class ConfusionMatrix { - double - tp = 0.0d, fn = 0.0d, - fp = 0.0d, tn = 0.0d; - } - - private void logsMatrices(List ms) { - final int n = ms.size(); - String[] putLine1 = new String[n]; - String[] putLine2 = new String[n]; - // Figure out width :/ - double max = 0.0d; - for (int i = 0; i < n; i++) { - ConfusionMatrix m = ms.get(i); - max = Math.max( - max, - Math.max( - Math.max(m.tp, m.fn), - Math.max(m.fp, m.tn))); - } - int w = (int) Math.floor(Math.log10(max)) + 4; - for (int i = 0; i < n; i++) { - putLine1[i] = String.format("[%" + w + ".2f %" + w + ".2f]", ms.get(i).tp, ms.get(i).fn); - putLine2[i] = String.format("[%" + w + ".2f %" + w + ".2f]", ms.get(i).fp, ms.get(i).tn); - } - LogInfo.log(Joiner.on(" ").join(putLine1)); - LogInfo.log(Joiner.on(" ").join(putLine2)); - } - - private void updateConfusionMatrix(ConfusionMatrix m, - Example ex, - double compatDecisionThreshold, - double probDecisionThreshold) { - List derivations = ex.getPredDerivations(); - double[] probs = Derivation.getProbs(derivations, 1.0d); - for (int i = 0; i < derivations.size(); i++) { - Derivation deriv = derivations.get(i); - double gold, pred; - if (compatDecisionThreshold == -1.0d) - gold = deriv.getCompatibility(); - else - gold = (deriv.getCompatibility() > compatDecisionThreshold) ? 1.0d : 0.0d; - if (probDecisionThreshold == -1.0d) - pred = probs[i]; - else - pred = (probs[i] > probDecisionThreshold) ? 1.0d : 0.0d; - m.tp += gold * pred; - m.fn += gold * (1.0d - pred); - m.fp += (1.0d - gold) * pred; - m.tn += (1.0d - gold) * (1.0d - pred); - } - } - - public void logsAll() { - boolean done = false; - for (int iter = 0; !done; iter++) - for (String group : new String[]{"train", "dev"}) - if (done = !logs(iter, group)) - break; - } -} diff --git a/src/edu/stanford/nlp/sempre/vis/ExampleDerivations.java b/src/edu/stanford/nlp/sempre/vis/ExampleDerivations.java deleted file mode 100644 index b51f42a..0000000 --- a/src/edu/stanford/nlp/sempre/vis/ExampleDerivations.java +++ /dev/null @@ -1,260 +0,0 @@ -package edu.stanford.nlp.sempre.vis; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import edu.stanford.nlp.sempre.Derivation; -import edu.stanford.nlp.sempre.Example; -import edu.stanford.nlp.sempre.Params; -import edu.stanford.nlp.sempre.Vis; -import fig.basic.IOUtils; -import fig.basic.LogInfo; -import fig.basic.MapUtils; -import fig.basic.Pair; -import fig.exec.Execution; - -import java.io.File; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Visualize predicate derivations from one or more executions of the semantic - * parser. - * - * @author Roy Frostig - */ -public class ExampleDerivations { - private final List execPaths; - private final int topN; - - public ExampleDerivations(List execPaths, int topN) { - this.execPaths = execPaths; - this.topN = topN; - } - - private PrintWriter tmpLog; - private void pushLog(PrintWriter newLog) { - tmpLog = LogInfo.getFileOut(); - LogInfo.setFileOut(newLog); - } - private void popLog() { - LogInfo.setFileOut(tmpLog); - } - - public boolean write(int iter, String group) { - List files = Vis.getFilesPerExec(execPaths, iter, group); - - if (files == null) - return false; - - LogInfo.logs("Reading files: %s", files); - List params = getParamsPerExec(iter); - assert files.size() == params.size(); - - String basePath = "vis.preds-iter" + iter + "-" + group + ".examples"; - String outPath = Execution.getFile(basePath); - PrintWriter out = IOUtils.openOutHard(outPath); - LogInfo.log("Writing " + basePath); - pushLog(out); - - int i = 0; - for (List row : Vis.zipExamples(files)) { - Example first = row.get(0); - LogInfo.begin_track("Example %d: %s [%s]", i, first.getUtterance(), first.getId()); - List> rowDerivIndices = getRowDerivIndices(row); - List firstDerivs = first.getPredDerivations(); - - LogInfo.logs("STAT %s", first.getEvaluation().summary()); - - for (int j = 0; j < firstDerivs.size(); j++) { - Derivation firstDeriv = firstDerivs.get(j); - LogInfo.begin_track("Ex %d, derivation %d", i, j); - if (firstDeriv.getCompatibility() == 1.0d) - LogInfo.log("DERIV CORRECT"); - else - LogInfo.log("DERIV WRONG"); - LogInfo.logs("DERIV %s", firstDeriv); - - // TODO no executor stats being written/loaded at present. - //firstDeriv.getExecutorStats().logStats(""+j); - - List> topFeatures; - List> botFeatures; - if (row.size() >= 2) { - Integer derivIndex = rowDerivIndices.get(1).get(firstDeriv); - Derivation secondDeriv = (derivIndex == null) - ? null - : row.get(1).getPredDerivations().get(derivIndex); - topFeatures = getTopFeatures( - topN, - params.get(0), - params.get(1), - firstDeriv, - secondDeriv, - false); - botFeatures = getTopFeatures( - topN, - params.get(0), - params.get(1), - firstDeriv, - secondDeriv, - true); - } else { - topFeatures = getTopFeatures( - topN, - params.get(0), - null, - firstDeriv, - null, - false); - botFeatures = getTopFeatures( - topN, - params.get(0), - null, - firstDeriv, - null, - true); - } - - String[] positions = new String[row.size()]; - String[][] chart = new String[topFeatures.size() + botFeatures.size()][row.size()]; - String[][] totals = new String[4][row.size()]; - - for (int k = 0; k < row.size(); k++) { - Integer derivIndex = rowDerivIndices.get(k).get(firstDeriv); - - // Positions - positions[k] = String.format("%12s", (derivIndex == null) ? "~" : ("" + derivIndex)); - - // Features - // Walk down topFeatures and up botFeatures at the same time. - int n = Math.max(topFeatures.size(), botFeatures.size()); - for (int f = 0; f < n; f++) { - if (f < topFeatures.size()) { - String featureName = topFeatures.get(f).getFirst(); - double val = params.get(k).getWeight(featureName); - chart[f][k] = String.format("%12.4f", val); - } - if (f < botFeatures.size()) { - String featureName = botFeatures.get(f).getFirst(); - double val = params.get(k).getWeight(featureName); - chart[chart.length - 1 - f][k] = String.format("%12.4f", val); - } - } - - // Totals - if (derivIndex == null) { - totals[0][k] = totals[1][k] = totals[2][k] = totals[3][k] = String.format("%12s", "~"); - } else { - Derivation deriv = row.get(k).getPredDerivations().get(derivIndex); - totals[0][k] = String.format("%12.4f", deriv.getScore()); - totals[1][k] = String.format("%12.4f", deriv.getProb()); - totals[2][k] = String.format("%12.4f", deriv.getCompatibility()); - totals[3][k] = String.format("%12d", deriv.getMaxBeamPosition()); - } - } - LogInfo.log(""); - - LogInfo.logs("%-40s\t%12s%s", "POS", "", Joiner.on(' ').join(positions)); - LogInfo.log(""); - - for (int f = 0; f < chart.length; f++) { - List> tops = (f < topFeatures.size()) ? topFeatures : botFeatures; - int topsIndex = (f < topFeatures.size()) ? f : chart.length - 1 - f; - if (f == topFeatures.size()) - LogInfo.log("..."); - LogInfo.logs( - "%-40s\t%12.4f%s", - "FEAT " + tops.get(topsIndex).getFirst(), - tops.get(topsIndex).getSecond(), - Joiner.on(' ').join(chart[f])); - } - LogInfo.log(""); - LogInfo.logs("%-40s\t%12s%s", "SCORE", "", Joiner.on(' ').join(totals[0])); - LogInfo.logs("%-40s\t%12s%s", "PROB", "", Joiner.on(' ').join(totals[1])); - LogInfo.logs("%-40s\t%12s%s", "COMPAT", "", Joiner.on(' ').join(totals[2])); - LogInfo.logs("%-40s\t%12s%s", "MAXBEAMPOS", "", Joiner.on(' ').join(totals[3])); - LogInfo.end_track(); - } - LogInfo.end_track(); - i++; - } - - popLog(); - return true; - } - - public void writeAll() { - boolean done = false; - for (int iter = 0; !done; iter++) - for (String group : new String[]{"train", "dev"}) - if (done = !write(iter, group)) - break; - } - - private List> getRowDerivIndices(List row) { - List> res = new ArrayList>(row.size()); - for (Example ex : row) { - Map m = new HashMap(); - int i = 0; - for (Derivation d : ex.getPredDerivations()) { - if (!m.containsKey(d)) - m.put(d, i); - i++; - } - res.add(m); - } - return res; - } - - private List> getTopFeatures(int topN, - Params firstParams, - Params secondParams, - Derivation firstDeriv, - Derivation secondDeriv, - boolean reverse) { - Map sortBy; - Map firstFeats = new HashMap(); - firstDeriv.incrementAllFeatureVector(1.0d, firstFeats); - double factor = reverse ? -1.0d : 1.0d; - if (secondDeriv == null) { - sortBy = Utils.scale( - factor, - Utils.elementwiseProduct( - firstFeats, - firstParams.getWeights())); - } else { - Map secondFeats = new HashMap(); - secondDeriv.incrementAllFeatureVector(1.0d, secondFeats); - sortBy = Utils.linearComb( - factor, -factor, - Utils.elementwiseProduct( - firstFeats, - firstParams.getWeights()), - Utils.elementwiseProduct( - secondFeats, - secondParams.getWeights())); - } - List> top = MapUtils.getTopN(sortBy, topN); - List> topFeats = Lists.newArrayListWithExpectedSize(top.size()); - for (Pair pair : top) { - topFeats.add( - Pair.newPair( - pair.getFirst(), - firstFeats.get(pair.getFirst()))); - } - return topFeats; - } - - private List getParamsPerExec(int iter) { - List params = new ArrayList(); - for (String execPath : execPaths) { - Params p = new Params(); - p.read(execPath + "/params." + iter); - params.add(p); - } - return params; - } -} diff --git a/src/edu/stanford/nlp/sempre/vis/Utils.java b/src/edu/stanford/nlp/sempre/vis/Utils.java deleted file mode 100644 index d570222..0000000 --- a/src/edu/stanford/nlp/sempre/vis/Utils.java +++ /dev/null @@ -1,119 +0,0 @@ -package edu.stanford.nlp.sempre.vis; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -/** - * Things that should go into Fig or similar. - * - * @author Roy Frostig - */ -public final class Utils { - private Utils() {} - - public static Map elementwiseProduct(Map a, Map b) { - Map m = new HashMap(); - for (Map.Entry entry : a.entrySet()) { - Double bBox = b.get(entry.getKey()); - double bVal = (bBox == null) ? 0.0d : bBox; - m.put(entry.getKey(), entry.getValue() * bVal); - } - return m; - } - - public static Map linearComb(double aFactor, double bFactor, - Map a, Map b) { - Map m = new HashMap(); - for (Map.Entry entry : a.entrySet()) { - Double bBox = b.get(entry.getKey()); - double bVal = (bBox == null) ? 0.0d : bBox; - m.put(entry.getKey(), aFactor * entry.getValue() + bFactor * bVal); - } - return m; - } - - public static Map scale(double factor, Map x) { - return linearComb(factor, 0.0d, x, new HashMap()); - } - - public static String readLineHard(BufferedReader in) { - try { - return in.readLine(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public static void closeHard(Closeable c) { - try { - c.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public static int[] resize(int[] arr, int n) { - int[] next = new int[n]; - int len = Math.min(arr.length, n); - for (int i = 0; i < len; i++) - next[i] = arr[i]; - return next; - } - - public static Map indicesOf(Iterable xs) { - Map m = new HashMap(); - int i = 0; - for (T x : xs) { - if (!m.containsKey(x)) - m.put(x, i); - i++; - } - return m; - } - - /** - * Simple generator-like iterable that provides objects of type T until the - * first null value. You only implement `computeNext()`. - */ - public static abstract class SimpleGenerator implements Iterable { - /** Return null when done. */ - abstract protected T computeNext(); - - public Iterator iterator() { - return new Iterator() { - private T buf = null; - - @Override - public boolean hasNext() { - // Either the item is already buffered (so we keep it - // buffered), or we are buffering one, or there is truly - // nothing to buffer. - buf = next(); - return buf != null; - } - - @Override - public T next() { - // If we have an item buffered, return it and clear buffer. - if (buf != null) { - T tmp = buf; - buf = null; - return tmp; - } - - // Otherwise, actually give us a fresh next item. - return computeNext(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - } -} diff --git a/testng.xml b/testng.xml index 64bc283..748d0a2 100644 --- a/testng.xml +++ b/testng.xml @@ -1,9 +1,14 @@ - + - + + + + + + diff --git a/unittest-files/README b/unittest-files/README new file mode 100644 index 0000000..3997f8a --- /dev/null +++ b/unittest-files/README @@ -0,0 +1 @@ +This directory contains small files which are read by unit tests. diff --git a/unittest-files/binaryInfoStringAndAlignment.txt b/unittest-files/binaryInfoStringAndAlignment.txt new file mode 100644 index 0000000..4b5e33d --- /dev/null +++ b/unittest-files/binaryInfoStringAndAlignment.txt @@ -0,0 +1,5 @@ +{"formula":"(lambda x (fb:people.person.education (fb:education.education.institution (var x))))","source":"ALIGNMENT","features":{"FB_typed_size":1740.0,"Intersection_size_typed":53.0,"NL-size":18096.0,"NL_typed_size":65.0},"lexeme":"bear in"} +{"formula":"fb:people.deceased_person.place_of_cremation","source":"ALIGNMENT","features":{"FB_typed_size":3.0,"Intersection_size_typed":1.0,"NL-size":18096.0,"NL_typed_size":7156.0},"lexeme":"bear in"} +{"formula":"fb:location.location.people_born_here","source":"ALIGNMENT","features":{"FB_typed_size":351.0,"Intersection_size_typed":1.0,"NL-size":18096.0,"NL_typed_size":1.0},"lexeme":"bear in"} +{"formula":"!fb:location.location.people_born_here","source":"ALIGNMENT","features":{"FB_typed_size":16184.0,"Intersection_size_typed":13856.0,"NL-size":18096.0,"NL_typed_size":15765.0},"lexeme":"bear in"} +{"formula":"(lambda x (!fb:tv.tv_producer_term.producer (!fb:tv.tv_program.tv_producer (var x))))","source":"ALIGNMENT","features":{"FB_typed_size":156.0,"Intersection_size_typed":1.0,"NL-size":10.0,"NL_typed_size":1.0},"lexeme":"hilarious in"} diff --git a/unittest-files/unaryInfoStringAndAlignment.txt b/unittest-files/unaryInfoStringAndAlignment.txt new file mode 100644 index 0000000..6f657e1 --- /dev/null +++ b/unittest-files/unaryInfoStringAndAlignment.txt @@ -0,0 +1,4 @@ +{"formula":"(fb:people.person.profession fb:en.attorney)","source":"STRING_MATCH","features":{},"lexeme":"lawyer"} +{"formula":"(fb:people.person.profession fb:en.attorney)","source":"ALIGNMENT","features":{"fb_size":847.0,"intersection":26.0,"nl_size":57.0},"lexeme":"lawyer"} +{"formula":"(fb:type.object.type fb:location.continent)","source":"STRING_MATCH","features":{},"lexeme":"continent"} +{"formula":"(fb:type.object.type fb:location.continent)","source":"ALIGNMENT","features":{"fb_size":6.0,"intersection":5.0,"nl_size":15.0},"lexeme":"continent"}