mirror of https://github.com/percyliang/sempre
Added tables and overnight modules
This commit is contained in:
Panupong Pasupat
parent
b1cde3658a
commit
9178eb7322
|
|
@ -0,0 +1,23 @@
|
|||
lib
|
||||
fig
|
||||
virtuoso-opensource
|
||||
module-classes.txt
|
||||
classes
|
||||
libsempre
|
||||
|
||||
state
|
||||
out
|
||||
test-output
|
||||
|
||||
.project
|
||||
.classpath
|
||||
.settings
|
||||
.idea
|
||||
semparse.iml
|
||||
*~
|
||||
*.swp
|
||||
*.bak
|
||||
*.pyc
|
||||
*.cache
|
||||
*.DS_Store
|
||||
java.hprof.txt
|
||||
38
Makefile
38
Makefile
|
|
@ -1,38 +0,0 @@
|
|||
BUILD_DEPS = libsempre/sempre-core.jar \
|
||||
libsempre/sempre-cache.jar \
|
||||
libsempre/sempre-freebase.jar \
|
||||
libsempre/sempre-fbalignment.jar \
|
||||
libsempre/sempre-paraphrase.jar \
|
||||
libsempre/sempre-corenlp.jar \
|
||||
libsempre/sempre-jungle.jar
|
||||
|
||||
|
||||
default: module-classes $(BUILD_DEPS)
|
||||
|
||||
module-classes:
|
||||
scripts/extract-module-classes.rb
|
||||
|
||||
core: libsempre/sempre-core.jar
|
||||
libsempre/sempre-core.jar: \
|
||||
$(shell ls src/edu/stanford/nlp/sempre/*.java) \
|
||||
$(shell ls src/edu/stanford/nlp/sempre/test/*.java)
|
||||
cd src/edu/stanford/nlp/sempre && ant compile
|
||||
|
||||
cache: libsempre/sempre-cache.jar
|
||||
libsempre/sempre-cache.jar: \
|
||||
$(shell find src/edu/stanford/nlp/sempre/cache -name "*.java")
|
||||
cd src/edu/stanford/nlp/sempre/cache && ant compile
|
||||
|
||||
corenlp: libsempre/sempre-corenlp.jar
|
||||
libsempre/sempre-corenlp.jar: libsempre/sempre-core.jar libsempre/sempre-cache.jar \
|
||||
$(shell find src/edu/stanford/nlp/sempre/corenlp -name "*.java")
|
||||
cd src/edu/stanford/nlp/sempre/corenlp && ant compile
|
||||
|
||||
freebase: libsempre/sempre-freebase.jar
|
||||
libsempre/sempre-freebase.jar: libsempre/sempre-core.jar libsempre/sempre-cache.jar \
|
||||
$(shell find src/edu/stanford/nlp/sempre/freebase -name "*.java")
|
||||
cd src/edu/stanford/nlp/sempre/freebase && ant compile
|
||||
|
||||
|
||||
clean:
|
||||
rm -rf classes libsempre
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# SEMPRE 2.0: Semantic Parsing with Execution
|
||||
# SEMPRE 2.1: Semantic Parsing with Execution
|
||||
|
||||
## What is semantic parsing?
|
||||
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ derivation representing a number.
|
|||
Now, you can parse the following:
|
||||
|
||||
four
|
||||
2.718
|
||||
20
|
||||
|
||||
Note: if you now type in `three`, you should get two derivations that yield the
|
||||
same answer, one coming from each rule. Note that `twenty-five million` will
|
||||
|
|
@ -334,7 +334,7 @@ The complete derivation for *three plus four* is illustrated here:
|
|||
three plus four
|
||||
|
||||
|
||||
**Exercise 2.1**: write rules can parse the following utterances into
|
||||
**Exercise 2.1**: write rules that can parse the following utterances into
|
||||
into the category `$Expr`:
|
||||
|
||||
length of hello world # 11
|
||||
|
|
@ -375,7 +375,7 @@ You can put a set of grammar rules in a file (e.g.,
|
|||
|
||||
./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar
|
||||
|
||||
If you make edit the grammar, you can reload the grammar without exiting the
|
||||
If you edit the grammar, you can reload the grammar without exiting the
|
||||
prompt by typing:
|
||||
|
||||
(reload)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project default="compile" name="semparse">
|
||||
<!-- Set useful variables -->
|
||||
<property name="target" value="1.8"/>
|
||||
<property name="source" value="1.8"/>
|
||||
<property name="root" value="."/>
|
||||
<property name="src" location="${root}/src/"/>
|
||||
<property name="classes" location="${root}/classes"/>
|
||||
<property name="lib" location="${root}/lib"/>
|
||||
<property name="libsempre" location="${root}/libsempre"/>
|
||||
<path id="lib.path">
|
||||
<fileset dir="${libsempre}" includes="*.jar"/>
|
||||
<fileset dir="${lib}" includes="*.jar"/>
|
||||
</path>
|
||||
|
||||
<!-- Create directories -->
|
||||
<target name="init">
|
||||
<exec executable="${root}/scripts/extract-module-classes.rb"/>
|
||||
<mkdir dir="${classes}"/>
|
||||
<mkdir dir="${libsempre}"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile -->
|
||||
<target name="compile" depends="init">
|
||||
<antcall target="compile.released"/>
|
||||
</target>
|
||||
|
||||
<target name="compile.released" depends="init,core,cache,corenlp,freebase,tables,overnight"/>
|
||||
|
||||
<!-- Compile core -->
|
||||
<target name="core" depends="init">
|
||||
<echo message="Compiling ${ant.project.name}: core"/>
|
||||
<mkdir dir="${classes}"/>
|
||||
<javac srcdir="${src}" destdir="${classes}" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/*.java"/>
|
||||
<include name="edu/stanford/nlp/sempre/test/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-core.jar" basedir="${classes}" includes="edu/**"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile cache -->
|
||||
<target name="cache" depends="init">
|
||||
<echo message="Compiling ${ant.project.name}: cache"/>
|
||||
<mkdir dir="${classes}/cache"/>
|
||||
<javac srcdir="${src}" destdir="${classes}/cache" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/cache/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-cache.jar" basedir="${classes}/cache"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile corenlp -->
|
||||
<target name="corenlp" depends="init,core,cache">
|
||||
<echo message="Compiling ${ant.project.name}: corenlp"/>
|
||||
<mkdir dir="${classes}/corenlp"/>
|
||||
<javac srcdir="${src}" destdir="${classes}/corenlp" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/corenlp/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-corenlp.jar" basedir="${classes}/corenlp"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile freebase -->
|
||||
<target name="freebase" depends="init,core,cache">
|
||||
<echo message="Compiling ${ant.project.name}: freebase"/>
|
||||
<mkdir dir="${classes}/freebase"/>
|
||||
<javac srcdir="${src}" destdir="${classes}/freebase" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/freebase/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-freebase.jar" basedir="${classes}/freebase"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile tables -->
|
||||
<target name="tables" depends="init,core">
|
||||
<echo message="Compiling ${ant.project.name}: tables"/>
|
||||
<mkdir dir="${classes}/tables"/>
|
||||
<javac srcdir="${src}" destdir="${classes}/tables" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/tables/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-tables.jar" basedir="${classes}/tables"/>
|
||||
</target>
|
||||
|
||||
<!-- Compile overnight -->
|
||||
<target name="overnight" depends="init,core">
|
||||
<echo message="Compiling ${ant.project.name}: overnight"/>
|
||||
<mkdir dir="${classes}/overnight"/>
|
||||
<javac srcdir="${src}" destdir="${classes}/overnight" classpathref="lib.path" debug="true" includeantruntime="false" source="${source}" target="${target}">
|
||||
<include name="edu/stanford/nlp/sempre/overnight/"/>
|
||||
</javac>
|
||||
<jar destfile="${libsempre}/sempre-overnight.jar" basedir="${classes}/overnight"/>
|
||||
</target>
|
||||
|
||||
|
||||
<!-- Clean up -->
|
||||
<target name="clean">
|
||||
<delete includeemptydirs="true" quiet="true">
|
||||
<fileset dir="${classes}" followsymlinks="false"/>
|
||||
<fileset dir="${libsempre}" followsymlinks="false"/>
|
||||
<fileset file="module-classes.txt" followsymlinks="false"/>
|
||||
</delete>
|
||||
</target>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Files for Building a Semantic Parser Overnight
|
||||
|
||||
## Setting up datasets
|
||||
|
||||
To generate canonical utterances, run
|
||||
|
||||
./pull-dependencies freebase overnight
|
||||
make corenlp freebase overnight
|
||||
./run @mode=genovernight-wrapper
|
||||
|
||||
Or, to generate each individual domain
|
||||
./run @mode=genovernight @gen=1 @domain=<domain>
|
||||
|
||||
After turking and setting up the approprate example files, run
|
||||
|
||||
./run @mode=overnight @domain=<domain>
|
||||
|
||||
to run with all features.
|
||||
|
||||
To run with partial features, use the following commands:
|
||||
|
||||
Baseline - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match skip-bigram root lf simpleworld
|
||||
No Lexical features - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root lf simpleworld
|
||||
No PPDB - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match skip-bigram root lf alignment lexical root_lexical simpleworld
|
||||
Full system - ./run @mode=overnight @domain=<domain> -OvernightFeatureComputer.featureDomains match ppdb skip-bigram root alignment lexical root_lexical lf simpleworld
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (player) (ConstantFn en.player))
|
||||
(rule $EntityNP1 (kobe bryant) (ConstantFn en.player.kobe_bryant))
|
||||
(rule $EntityNP2 (lebron james) (ConstantFn en.player.lebron_james))
|
||||
|
||||
(rule $TypeNP (team) (ConstantFn en.team))
|
||||
(rule $EntityNP1 (los angeles lakers) (ConstantFn en.team.lakers))
|
||||
(rule $EntityNP2 (cleveland cavaliers) (ConstantFn en.team.cavaliers))
|
||||
|
||||
(rule $TypeNP (position) (ConstantFn en.position))
|
||||
(rule $EntityNP1 (point guard) (ConstantFn en.position.point_guard))
|
||||
(rule $EntityNP2 (forward) (ConstantFn en.position.forward))
|
||||
|
||||
(rule $EntityNP1 (2004) (ConstantFn (date 2004 -1 -1)))
|
||||
(rule $EntityNP2 (2010) (ConstantFn (date 2010 -1 -1)))
|
||||
|
||||
(for @x (point assist steal turnover rebound block foul game fg ft)
|
||||
(rule $EntityNP1 (3) (ConstantFn (number 3 @x)))
|
||||
#(rule $EntityNP2 (6) (ConstantFn (number 6 @x)))
|
||||
)
|
||||
|
||||
# Season statistics
|
||||
|
||||
(rule $Rel0NP (player) (ConstantFn (string player)))
|
||||
(rule $RelNP (position) (ConstantFn (string position)))
|
||||
(rule $RelNP (team) (ConstantFn (string team)))
|
||||
(rule $RelNP (season) (ConstantFn (string season)))
|
||||
|
||||
(rule $RelNP (number of points "(over a season)") (ConstantFn (string num_points)))
|
||||
(rule $RelNP (number of assists "(over a season)") (ConstantFn (string num_assists)))
|
||||
(rule $RelNP (number of steals "(over a season)") (ConstantFn (string num_steals)))
|
||||
(rule $RelNP (number of turnovers "(over a season)") (ConstantFn (string num_turnovers)))
|
||||
(rule $RelNP (number of rebounds "(over a season)") (ConstantFn (string num_rebounds)))
|
||||
(rule $RelNP (number of blocks "(over a season)") (ConstantFn (string num_blocks)))
|
||||
(rule $RelNP (number of fouls "(over a season)") (ConstantFn (string num_fouls)))
|
||||
(rule $RelNP (number of played games "(over a season)") (ConstantFn (string num_games_played)))
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (block) (ConstantFn en.block))
|
||||
(rule $EntityNP1 (block 1) (ConstantFn en.block.block1))
|
||||
(rule $EntityNP2 (block 2) (ConstantFn en.block.block2))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (shape) (ConstantFn (string shape)))
|
||||
(rule $EntityNP1 (a pyramid) (ConstantFn en.shape.pyramid))
|
||||
(rule $EntityNP2 (a cube) (ConstantFn en.shape.cube))
|
||||
|
||||
(rule $RelNP (color) (ConstantFn (string color)))
|
||||
(rule $EntityNP1 (red) (ConstantFn en.color.red))
|
||||
(rule $EntityNP2 (green) (ConstantFn en.color.green))
|
||||
|
||||
(rule $RelNP (length) (ConstantFn (string length)))
|
||||
(rule $RelNP (width) (ConstantFn (string width)))
|
||||
(rule $RelNP (height) (ConstantFn (string height)))
|
||||
(rule $EntityNP1 (3 inches) (ConstantFn (number 3 en.inch)))
|
||||
(rule $EntityNP2 (6 inches) (ConstantFn (number 6 en.inch)))
|
||||
|
||||
(rule $VP/NP (is left of) (ConstantFn (string left)))
|
||||
(rule $VP/NP (is right of) (ConstantFn (string right)))
|
||||
(rule $VP/NP (is above) (ConstantFn (string above)))
|
||||
(rule $VP/NP (is below) (ConstantFn (string below)))
|
||||
|
||||
(rule $VP (is special) (ConstantFn (string is_special)))
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# Make sure these test cases always parse
|
||||
(example (utterance "create a meeting with start time 3pm"))
|
||||
(example (utterance "what is the meeting whose start time is 9am or 3pm"))
|
||||
(example (utterance "remove the meeting whose date is day 2"))
|
||||
(example (utterance "what is the location of the meeting whose start time is 9am"))
|
||||
(example (utterance "what is the length of the free block whose date is day 1 with the largest end time"))
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (meeting) (ConstantFn en.meeting))
|
||||
(rule $EntityNP1 (weekly standup) (ConstantFn en.meeting.weekly_standup))
|
||||
(rule $EntityNP2 (annual review) (ConstantFn en.meeting.annual_review))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (date) (ConstantFn (string date)))
|
||||
(rule $EntityNP1 (jan 2) (ConstantFn (date 2015 1 2)))
|
||||
(rule $EntityNP2 (jan 3) (ConstantFn (date 2015 1 3)))
|
||||
|
||||
(rule $RelNP (start time) (ConstantFn (string start_time)))
|
||||
(rule $RelNP (end time) (ConstantFn (string end_time)))
|
||||
(rule $EntityNP1 (10am) (ConstantFn (time 10 0)))
|
||||
(rule $EntityNP2 (3pm) (ConstantFn (time 15 0)))
|
||||
|
||||
(rule $RelNP (length) (ConstantFn (string length)))
|
||||
(rule $EntityNP1 (three hours) (ConstantFn (number 3 en.hour)))
|
||||
(rule $EntityNP2 (one hour) (ConstantFn (number 1 en.hour)))
|
||||
|
||||
(rule $RelNP (attendee) (ConstantFn (string attendee)))
|
||||
(rule $TypeNP (person) (ConstantFn en.person))
|
||||
(rule $EntityNP1 (alice) (ConstantFn en.person.alice))
|
||||
(rule $EntityNP2 (bob) (ConstantFn en.person.bob))
|
||||
|
||||
(rule $RelNP (location) (ConstantFn (string location)))
|
||||
(rule $TypeNP (location) (ConstantFn en.location))
|
||||
(rule $EntityNP1 (greenberg cafe) (ConstantFn en.location.greenberg_cafe))
|
||||
(rule $EntityNP2 (central office) (ConstantFn en.location.central_office))
|
||||
|
||||
# Unaries
|
||||
(rule $VP (is important) (ConstantFn (string is_important)))
|
||||
|
|
@ -0,0 +1,428 @@
|
|||
(example
|
||||
(utterance "What meeting is on Jan 3?")
|
||||
(original "meeting whose date is jan 3")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 1 3)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many meetings after 1pm?")
|
||||
(original "number of meeting whose start time is larger than 1pm")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string start_time)) (string >) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericEntity (time 13 0)))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What meeting is on Wed Jan 5")
|
||||
(original "meeting whose date is jan 5")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 1 5)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Am I free on mar 6?")
|
||||
(original "number of meeting whose date is mar 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string date)) (string =) (date 2015 3 6))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Which meetings are about the office chair?")
|
||||
(original "meeting whose subject is office chair")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string subject) (string =) (en.subject.office_chair)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What time does the office meeting start?")
|
||||
(original "start time of meeting whose location is office")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string location) (string =) en.location.office) (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Who is attending the meeting on June 9?")
|
||||
(original "attendee of meeting whose date is jun 9")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 9)) (string attendee)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How long is the meeting on May 23?")
|
||||
(original "length of meeting whose date is may 23")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 23)) (string length)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Where is the meeting on Jun 12?")
|
||||
(original "location of meeting whose date is jun 12")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 12)) (string location)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What time is my weekly standup?")
|
||||
(original "start time of weekly standup")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty en.meeting.weekly_standup (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many times have I visited greenburg cafe at noon?")
|
||||
(original "Number of meetings whose location is greenburg cafe and whose start time is 12pm")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string location) (string =) en.location.greenburg_cafe) (string start_time) (string =) (time 12 0))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Are there any meetings in June?")
|
||||
(original "number of meeting whose date is at least jun 1 and whose date is at most jun 30")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string >=) (date 2015 6 1)) (string date) (string <=) (date 2015 6 30))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Do I have meetings on May 8?")
|
||||
(original "number of meeting whose date is may 8")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld. getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string date)) (string =) (date 2015 5 8))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What do I have scheduled on May 7?")
|
||||
(original "meeting whose date may 7")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 7)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many meetings does Alice have on May 12?")
|
||||
(original "Number of meetings whose attendee is alice and whose date is may 12")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string attendee) (string =) en.person.alice) (string date) (string =) (date 2015 5 12))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Who is attending the same meetings as me on May 7th?")
|
||||
(original "attendee of meeting whose date is may 7")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 7)) (string attendee)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Who is attending the 2:00 pm meeting on January 3rd?")
|
||||
(original "attendee of meeting whose start time is 2pm and whose date is jan 3)
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string start_time) (string =) (time 14 0)) (string date) (string =) (date 2015 1 3)) (string attendee)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What time does my first meeting start on May 17?")
|
||||
(original "start time of meeting whose date is may 17 that has smallest start time")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.superlative (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 17)) (string min) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string start_time))) (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many people will be attending the meeting on December 23?")
|
||||
(original "number of attendee of meeting whose date is dec 23")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 12 23)) (string attendee))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Where are my meetings on Feb 3?")
|
||||
(original "location of meeting whose date is feb 3")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 2 3)) (string location)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Who will be at the meeting on Feb 3?")
|
||||
(original "attendee of meeting whose date is feb 3")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 2 3)) (string attendee)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "When is my first meeting on Feb 3?")
|
||||
(original "start time of meeting whose date is feb 3 that has smallest start time")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.superlative (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 2 3)) (string min) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string start_time))) (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "When is my last meeting on Feb 3?")
|
||||
(original "start time of meeting whose date is feb 3 that has largest start time")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.superlative (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 2 3)) (string max) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string start_time))) (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Who is attending the meeting on My6?")
|
||||
(original "attendee of meeting whose date is may 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 6)) (string attendee)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Where is the location of the meeting on May 7?")
|
||||
(original "location of meeting whose date is may 7")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 7)) (string location)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What is the subject of my June 6 meeting?")
|
||||
(original "subject of meeting whose date is jun 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 6)) (string subject)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Where is my June 6 meeting?")
|
||||
(original "location of meeting whose date is jun 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 6)) (string location)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance Who is attending my June 6 meeting?")
|
||||
(original "attendee of meeting whose date is jun 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 6)) (string attendee)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What is the subject of my June 6 meeting?")
|
||||
(original "subject of meeting whose date is jun 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 6)) (string subject)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What time does the June 6 meeting start?")
|
||||
(original "start time of meeting whose date is jun 6")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 6)) (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Which meetings are in the afternoon?)
|
||||
(original "meeting whose start time is at least 12pm")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string start_time)) (string >=) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericEntity (time 12 0)))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many meetings are on Dec 12?")
|
||||
(original "number of meeting whose date is dec 12")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld. getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string date)) (string =) (date 2015 12 12))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many meetings is Alice going to in April?")
|
||||
(original "number of meeting whose attendee is alice and whose date is at least apr 1 and whose date is at most apr 30")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string attendee) (string =) en.person.alice) (string date) (string >=) (date 2015 4 1)) (string date) (string <=) (date 2015 4 30))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What is the longest meeting scheduled for the week of May 4-8?")
|
||||
(original "meeting whose date is at least may 4 and whose date is at most may 8 that has the largest length")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.superlative (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string >=) (date 2015 5 4)) (string date) (string <=) (date 2015 5 8)) (string max) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string length))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Do I have any meetings on May 8th?")
|
||||
(original "number of meeting whose date is may 8")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld. getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (call edu.stanford.nlp.sempre.agile.SimpleWorld.ensureNumericProperty (string date)) (string =) (date 2015 5 8))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "When does my meeting on June 3rd end?")
|
||||
(original "end time of meeting whose date is jun 3")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 6 3)) (string end_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "What time is my meeting on May 7th?")
|
||||
(original "start time of meeting whose date is may 7")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 7)) (string start_time)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many people will be in the May 10th meeting?")
|
||||
(original "number of attendee of meeting whose date is may 10")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call @getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 5 10)) (string attendee))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Do I have any important meetings in April?")
|
||||
(original "number of meeting that is important and whose date is at least apr 1 and whose date is at most april 30")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (cal l edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string is_important)) (string date) (string >=) (date 2015 4 1)) (string date) (string <=) (date 2015 4 30))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "How many important meetings do I have in April?")
|
||||
(original "number of meeting that is important and whose date is at least apr 1 and whose date is at most april 30")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (cal l edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string is_important)) (string date) (string >=) (date 2015 4 1)) (string date) (string <=) (date 2015 4 30))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Will Alice be at the meeting on May 10th?")
|
||||
(original "meeting whose attendee is alice and whose date is may 10")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call .size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string attendee) (string =) en.person.alice) (string date) (string =) (date 2015 5 10))))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Is there a meeting on February 3?")
|
||||
(original "meeting whose date is feb 3")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 2 3)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "Which meetings are scheduled for August 1?")
|
||||
(original "meeting whose date is aug 1")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (call edu.stanford.nlp.sempre.agile.SimpleWorld.getProperty (call edu.stanford.nlp.sempre.agile.SimpleWorld.singleton en.meeting) (string !type)) (string date) (string =) (date 2015 8 1)))
|
||||
)
|
||||
)
|
||||
|
||||
(example
|
||||
(utterance "how many meetings do I have to attend in July?")
|
||||
(original "number of meeting whose date is at least aug 1 and whose date is at most aug 31")
|
||||
(targetFormula
|
||||
(call edu.stanford.nlp.sempre.agile.SimpleWorld.listValue (call.size (call edu.stanford.nlp.sempre.agile.SimpleWorld.filter (edu.stanford.nlp.sempre.agile.SimpleWorld.)
|
||||
)
|
||||
)
|
||||
how many meetings do I have to attend in July?
|
||||
where is the 2pm meeting being held?
|
||||
|
||||
|
||||
Which meetings in May 1-7 are in the morning?
|
||||
Who is attending the May 19th meeting?
|
||||
which is my next day without a meeting?
|
||||
When is my next meeting with Alice?
|
||||
What is happening on the third?
|
||||
when does my last meeting end on feb 14?
|
||||
how many people will be at the march 4 meeting?
|
||||
What time is my meeting on Monday May 11th?
|
||||
Do I have anything scheduled on Wednesday May 13th?
|
||||
where is the last meeting of the year?
|
||||
how many meeting are in February?
|
||||
How many meetings are after noon?
|
||||
How many meetings are after noon on May 9th?
|
||||
How many meetings are before 4 pm on May 10th?
|
||||
Which meeting is Jami attending on May 11th?
|
||||
What is the agenda of the meeting?
|
||||
How long will the meeting prolong?
|
||||
How many important meetings do I have before noon on May 9th?
|
||||
What meetings are in March?
|
||||
how many meetings on jul 23 are longer than 1 hour?
|
||||
how many meetings have more than 2 attendees?
|
||||
what is the subject of the meeting on nov 23 at 4 pm?
|
||||
do i have any plans for 15th march
|
||||
can you check my appointments for the evening of the 5th
|
||||
What meetings are scheduled for May 5th?
|
||||
Where is the meeting scheduled for May 5th?
|
||||
How many will be attending the May 5th meeting?
|
||||
What time does the May 5th meeting start?
|
||||
what time is my first meeting on February 22?
|
||||
What meetings are scheduled with the CEO, Bob?
|
||||
What time does my last meeting end on June 22?
|
||||
Which meetings are about the annual budget review?
|
||||
what time is my next meeting with bob?
|
||||
|
||||
==== Requires "now"/"today"/"tomorrow" ====
|
||||
How many meetings do I have next week?
|
||||
Is Alice going to the meeting tomorrow?
|
||||
Where are the meetings for tomorrow?
|
||||
How many meetings do I have today?
|
||||
When do I have an important meeting in the next three days?
|
||||
What meetings are scheduled for next week?
|
||||
What time is tomorrow's meeting?
|
||||
How many meetings are this month?
|
||||
What is the earliest meeting tomorrow?
|
||||
What is the latest meeting tomorrow?
|
||||
What time is my next meeting?
|
||||
How many people will attend the meeting tomorrow?
|
||||
What time does today's meeting start?
|
||||
Where is my first meeting of the day at?
|
||||
When does my last meeting of the day end?
|
||||
Find the address for my next meeting?
|
||||
How much time do I have until my next meeting with Sam?
|
||||
How many meetings are today?
|
||||
Am I free at 2 pm today?
|
||||
How many meetings do I have this week?
|
||||
What is on today's calendar?
|
||||
When is my next standup?
|
||||
How many meetings are this month?
|
||||
|
||||
==== Implicit reference ====
|
||||
How many attendees are planning on going to the meeting?
|
||||
What time do I have to be at the meeting on May 9?
|
||||
|
||||
==== Not in grammar =====
|
||||
Is it possible to schedule a meeting before 2 pm?
|
||||
When do I call my boss Bob?
|
||||
When is my date with Alice?
|
||||
What meetings are with my Client Adam?
|
||||
Which month is most popular for meetings?
|
||||
When is the annual report due?
|
||||
|
||||
==== yes/no ====
|
||||
Do I have more than 2 meetings on April 5?
|
||||
|
|
@ -0,0 +1,416 @@
|
|||
############################################################
|
||||
# General grammar
|
||||
# There are two types of grammars
|
||||
# - generate: used to generate canonical utterances
|
||||
# - parse: used to actually parse
|
||||
|
||||
(def @domain edu.stanford.nlp.sempre.overnight.SimpleWorld.domain)
|
||||
(def @singleton edu.stanford.nlp.sempre.overnight.SimpleWorld.singleton)
|
||||
(def @filter edu.stanford.nlp.sempre.overnight.SimpleWorld.filter)
|
||||
(def @getProperty edu.stanford.nlp.sempre.overnight.SimpleWorld.getProperty)
|
||||
(def @superlative edu.stanford.nlp.sempre.overnight.SimpleWorld.superlative)
|
||||
(def @countSuperlative edu.stanford.nlp.sempre.overnight.SimpleWorld.countSuperlative)
|
||||
(def @countComparative edu.stanford.nlp.sempre.overnight.SimpleWorld.countComparative)
|
||||
(def @aggregate edu.stanford.nlp.sempre.overnight.SimpleWorld.aggregate)
|
||||
(def @concat edu.stanford.nlp.sempre.overnight.SimpleWorld.concat)
|
||||
(def @reverse edu.stanford.nlp.sempre.overnight.SimpleWorld.reverse)
|
||||
(def @arithOp edu.stanford.nlp.sempre.overnight.SimpleWorld.arithOp)
|
||||
(def @sortAndToString edu.stanford.nlp.sempre.overnight.SimpleWorld.sortAndToString)
|
||||
(def @ensureNumericProperty edu.stanford.nlp.sempre.overnight.SimpleWorld.ensureNumericProperty)
|
||||
(def @ensureNumericEntity edu.stanford.nlp.sempre.overnight.SimpleWorld.ensureNumericEntity)
|
||||
(def @listValue edu.stanford.nlp.sempre.overnight.SimpleWorld.listValue)
|
||||
|
||||
############################################################
|
||||
# Base cases
|
||||
|
||||
(when parse
|
||||
# G1
|
||||
# Generic values
|
||||
(rule $EntityNP ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $EntityNP ($PHRASE) (DateFn) (anchored 1))
|
||||
|
||||
# Currently, just cheat and use the entities defined in the base grammar.
|
||||
# In the future, want actually NER.
|
||||
(rule $EntityNP ($EntityNP1) (IdentityFn))
|
||||
(rule $EntityNP ($EntityNP2) (IdentityFn))
|
||||
#(rule $EntityNP ($PHRASE) (FilterNerSpanFn PERSON ORGANIZATION LOCATION MISC) (anchored 1))
|
||||
#(rule $EntityNP ($PHRASE) (FilterPosTagFn span NNP) (anchored 1))
|
||||
|
||||
(rule $Num ($PHRASE) (NumberFn) (anchored 1))
|
||||
)
|
||||
(when generate
|
||||
(rule $Num (two) (ConstantFn (number 2)))
|
||||
)
|
||||
|
||||
(when (and parse general)
|
||||
# G1
|
||||
(rule $NP ($EntityNP) (IdentityFn))
|
||||
# G2
|
||||
(rule $NP ($TypeNP) (lambda t (call @getProperty (call @singleton (var t)) (string !type)))) # Unary
|
||||
)
|
||||
(when (and generate general)
|
||||
(rule $UnaryNP ($TypeNP) (lambda t (call @getProperty (call @singleton (var t)) (string !type)))) # Unary
|
||||
)
|
||||
|
||||
(when (and parse regex)
|
||||
(rule $NP ($EntityNP) (IdentityFn))
|
||||
(rule $NP ($TypeNP) (IdentityFn))
|
||||
)
|
||||
(when (and generate regex)
|
||||
(rule $UnaryNP ($TypeNP) (IdentityFn))
|
||||
)
|
||||
|
||||
(when (and parse general)
|
||||
(rule $NumberRelNP ($RelNP) (lambda r (call @ensureNumericProperty (var r))))
|
||||
(rule $NumberNP ($NP) (lambda r (call @ensureNumericEntity (var r))))
|
||||
)
|
||||
(when (and generate general)
|
||||
(rule $NumberRelNP ($RelNP) (lambda r (call @ensureNumericProperty (var r))))
|
||||
(rule $NumberEntityNP ($EntityNP1) (lambda r (call @ensureNumericEntity (var r))))
|
||||
(rule $NumberNP0 ($NP0) (lambda r (call @ensureNumericEntity (var r))))
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Complementizer phrase (filtering)
|
||||
|
||||
(when (and parse general)
|
||||
# R0
|
||||
(rule $CP (that $VP) (lambda r (lambda s (call @filter (var s) (var r))))) # Vunary
|
||||
# R1
|
||||
(rule $CP (whose $RelNP is $NP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Req
|
||||
(rule $CP (whose $RelNP is not $NP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string !=) (var n)))))) # Rnot
|
||||
(rule $CP (whose $NumberRelNP is smaller than $NumberNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string <) (var n)))))) # Rl
|
||||
(rule $CP (whose $NumberRelNP is larger than $NumberNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string >) (var n)))))) # Rg
|
||||
(rule $CP (whose $NumberRelNP is at most $NumberNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string <=) (var n)))))) # Rle
|
||||
(rule $CP (whose $NumberRelNP is at least $NumberNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string >=) (var n)))))) # Rge
|
||||
# R2
|
||||
(rule $CP (that $VP/NP $NP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Vobj
|
||||
(rule $CP (that not $VP/NP $NP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string !=) (var n)))))) # Vobj-not
|
||||
# R3
|
||||
(rule $CP (that is $RelNP of $NP) (lambda r (lambda n (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Reqrev
|
||||
(rule $CP (that is not $RelNP of $NP) (lambda r (lambda n (lambda s (call @filter (var s) (call @reverse (var r)) (string !=) (var n)))))) # Reqrev-not
|
||||
# R4
|
||||
(rule $CP (that $NP $VP/NP) (lambda n (lambda r (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Vsubj
|
||||
(rule $CP (that $NP not $VP/NP) (lambda n (lambda r (lambda s (call @filter (var s) (call @reverse (var r)) (string !=) (var n)))))) # Vsubj-not
|
||||
)
|
||||
(when (and generate general)
|
||||
# R0
|
||||
(rule $CP00 (that $VP) (lambda r (lambda s (call @filter (var s) (var r))))) # Vunary
|
||||
# R1
|
||||
(rule $CP00 (whose $RelNP is $EntityNP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Req
|
||||
(rule $CP1 (whose $RelNP is $NP0) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Req
|
||||
(rule $CP0 (whose $RelNP is not $EntityNP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string !=) (var n)))))) # Rnot
|
||||
(rule $CP0 (whose $NumberRelNP is smaller than $NumberEntityNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string <) (var n)))))) # Rl
|
||||
(rule $CP1 (whose $NumberRelNP is smaller than $NumberNP0) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string <) (var n)))))) # Rl
|
||||
(rule $CP0 (whose $NumberRelNP is larger than $NumberEntityNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string >) (var n)))))) # Rg
|
||||
(rule $CP1 (whose $NumberRelNP is larger than $NumberNP0) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string >) (var n)))))) # Rg
|
||||
(rule $CP0 (whose $NumberRelNP is at most $NumberEntityNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string <=) (var n)))))) #Rle
|
||||
(rule $CP1 (whose $NumberRelNP is at most $NumberNP0) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string <=) (var n)))))) # Rle
|
||||
(rule $CP0 (whose $NumberRelNP is at least $NumberEntityNP) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string >=) (var n)))))) # Rge
|
||||
(rule $CP1 (whose $NumberRelNP is at least $NumberNP0) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string >=) (var n)))))) # Rge
|
||||
# R2
|
||||
(rule $CP00 (that $VP/NP $EntityNP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Vobj
|
||||
(rule $CP0 (that not $VP/NP $EntityNP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string !=) (var n)))))) # Vobj-neg
|
||||
(rule $CP1 (that $VP/NP $NP0) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Vobj
|
||||
# R3
|
||||
(rule $CP00 (that is $RelNP of $EntityNP1) (lambda r (lambda n (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Req2
|
||||
(rule $CP0 (that is not $RelNP of $EntityNP1) (lambda r (lambda n (lambda s (call @filter (var s) (call @reverse (var r)) (string !=) (var n)))))) # Req2
|
||||
(rule $CP1 (that is $RelNP of $NP0) (lambda r (lambda n (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Req2
|
||||
# R4
|
||||
(rule $CP00 (that $EntityNP1 $VP/NP) (lambda n (lambda r (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Vsubj
|
||||
(rule $CP0 (that $EntityNP1 not $VP/NP) (lambda n (lambda r (lambda s (call @filter (var s) (call @reverse (var r)) (string !=) (var n)))))) # Vsubj
|
||||
(rule $CP1 (that $NP0 $VP/NP) (lambda n (lambda r (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Vsubj
|
||||
)
|
||||
(when geo880
|
||||
(rule $CP2 (that $NP1 $VP/NP) (lambda n (lambda r (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Vsubj
|
||||
(rule $CP2 (that $VP/NP $NP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Vobj
|
||||
(rule $CP2 (that is $RelNP of $NP1) (lambda r (lambda n (lambda s (call @filter (var s) (call @reverse (var r)) (string =) (var n)))))) # Req2
|
||||
(rule $CP2 (whose $RelNP is $NP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Req
|
||||
)
|
||||
(when geo440
|
||||
(rule $CP2 (that $VP/NP $NP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Vobj
|
||||
(rule $CP2 (whose $RelNP is $NP1) (lambda r (lambda n (lambda s (call @filter (var s) (var r) (string =) (var n)))))) # Req
|
||||
)
|
||||
|
||||
(when (and parse regex)
|
||||
# I use modifier for negation, we did not have this negation in previous grammars
|
||||
(rule $Modifier ($VP/NP $NP) (JoinFn forward betaReduce))
|
||||
(rule $Modifier ($VP/NP1 $NP) (JoinFn forward betaReduce))
|
||||
(rule $CP (that $Modifier) (IdentityFn))
|
||||
(rule $CP (that not $Modifier) (lambda m (call + (string "~\(") (var m) (string "\)"))))
|
||||
)
|
||||
(when (and generate regex)
|
||||
(rule $Modifier0 ($VP/NP $EntityNP1) (JoinFn forward betaReduce))
|
||||
(rule $Modifier1 ($VP/NP $NP0) (JoinFn forward betaReduce))
|
||||
(rule $Modifier0 ($VP/NP1 $EntityNP1) (JoinFn forward betaReduce))
|
||||
(rule $Modifier1 ($VP/NP1 $NP0) (JoinFn forward betaReduce))
|
||||
(rule $CP00 (that $Modifier0) (IdentityFn))
|
||||
(rule $CP1 (that $Modifier1) (IdentityFn))
|
||||
(rule $CP1 (that not $Modifier0) (lambda m (call + (string "~\(") (var m) (string "\)"))))
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Complementizer phrase (superlatives and comparatives)
|
||||
|
||||
(when (and parse general)
|
||||
# S0
|
||||
(rule $CP (that has the smallest $NumberRelNP) (lambda r (lambda s (call @superlative (var s) (string min) (var r))))) # Smin
|
||||
(rule $CP (that has the largest $NumberRelNP) (lambda r (lambda s (call @superlative (var s) (string max) (var r))))) # Smax
|
||||
# S1
|
||||
(rule $CP (that has the least number of $RelNP) (lambda r (lambda s (call @countSuperlative (var s) (string min) (var r))))) # Scmin
|
||||
(rule $CP (that has the most number of $RelNP) (lambda r (lambda s (call @countSuperlative (var s) (string max) (var r))))) # Scmax
|
||||
# S2
|
||||
(rule $CP (that $VP/NP the least number of $NP) (lambda r (lambda s2 (lambda s1 (call @countSuperlative (var s1) (string min) (var r) (var s2)))))) # Scvmin
|
||||
(rule $CP (that $VP/NP the most number of $NP) (lambda r (lambda s2 (lambda s1 (call @countSuperlative (var s1) (string max) (var r) (var s2)))))) # Scvmax
|
||||
# S3
|
||||
(rule $CP (that is $RelNP of the least number of $NP) (lambda r (lambda np (lambda s (call @countSuperlative (var s) (string min) (call @reverse (var r)) (var np)))))) # Scmin
|
||||
(rule $CP (that is $RelNP of the most number of $NP) (lambda r (lambda np (lambda s (call @countSuperlative (var s) (string max) (call @reverse (var r)) (var np)))))) # Scmax
|
||||
# S4
|
||||
(rule $CP (that the least number of $NP $VP/NP) (lambda np (lambda r (lambda s (call @countSuperlative (var s) (string min) (call @reverse (var r)) (var np)))))) # Scvmin
|
||||
(rule $CP (that the most number of $NP $VP/NP) (lambda np (lambda r (lambda s (call @countSuperlative (var s) (string max) (call @reverse (var r)) (var np)))))) # Scvmax
|
||||
# C1
|
||||
(rule $CP (that has $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string =) (var num)))))) # Ceq
|
||||
(rule $CP (that has less than $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string <) (var num)))))) # Cl
|
||||
(rule $CP (that has more than $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string >) (var num)))))) # Cg
|
||||
(rule $CP (that has at most $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string <=) (var num)))))) # Cleq
|
||||
(rule $CP (that has at least $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string >=) (var num)))))) # Cgeq
|
||||
# C2
|
||||
(rule $CP (that $VP/NP $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string =) (var num) (var np))))))) # Ceq
|
||||
(rule $CP (that $VP/NP less than $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string <) (var num) (var np))))))) # Cl
|
||||
(rule $CP (that $VP/NP more than $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string >) (var num) (var np))))))) # Cg
|
||||
(rule $CP (that $VP/NP at most $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string <=) (var num) (var np))))))) # Cleq
|
||||
(rule $CP (that $VP/NP at least $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string >=) (var num) (var np))))))) # Cgeq
|
||||
# C3
|
||||
(rule $CP (that is $RelNP of $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string =) (var num) (var np))))))) # Ceq
|
||||
(rule $CP (that is $RelNP of less than $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <) (var num) (var np))))))) # Cl
|
||||
(rule $CP (that is $RelNP of more than $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >) (var num) (var np))))))) # Cg
|
||||
(rule $CP (that is $RelNP of at most $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <=) (var num) (var np))))))) # Cleq
|
||||
(rule $CP (that is $RelNP of at least $Num $NP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >=) (var num) (var np))))))) # Cgeq
|
||||
# C4
|
||||
(rule $CP (that $Num $NP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string =) (var num) (var np))))))) # Ceq
|
||||
(rule $CP (that less than $Num $NP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <) (var num) (var np))))))) # Cl
|
||||
(rule $CP (that more than $Num $NP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >) (var num) (var np))))))) # Cg
|
||||
(rule $CP (that at most $Num $NP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <=) (var num) (var np))))))) # Cleq
|
||||
(rule $CP (that at least $Num $NP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >=) (var num) (var np))))))) # Cgeq
|
||||
)
|
||||
(when (and generate general) # $CP => $CP1, $NP => $UnaryNP
|
||||
# S1
|
||||
(rule $CP1 (that has the smallest $NumberRelNP) (lambda r (lambda s (call @superlative (var s) (string min) (var r))))) # Smin
|
||||
(rule $CP1 (that has the largest $NumberRelNP) (lambda r (lambda s (call @superlative (var s) (string max) (var r))))) # Smax
|
||||
# S2
|
||||
(rule $CP1 (that has the least number of $RelNP) (lambda r (lambda s (call @countSuperlative (var s) (string min) (var r))))) # Scmin
|
||||
(rule $CP1 (that has the most number of $RelNP) (lambda r (lambda s (call @countSuperlative (var s) (string max) (var r))))) # Scmax
|
||||
# S3
|
||||
(rule $CP1 (that $VP/NP the least number of $UnaryNP) (lambda r (lambda s2 (lambda s1 (call @countSuperlative (var s1) (string min) (var r) (var s2)))))) # Scvmin
|
||||
(rule $CP1 (that $VP/NP the most number of $UnaryNP) (lambda r (lambda s2 (lambda s1 (call @countSuperlative (var s1) (string max) (var r) (var s2)))))) # Scvmax
|
||||
# S4
|
||||
(rule $CP1 (that is $RelNP of the least number of $UnaryNP) (lambda r (lambda np (lambda s (call @countSuperlative (var s) (string min) (call @reverse (var r)) (var np)))))) # Scmin
|
||||
(rule $CP1 (that is $RelNP of the most number of $UnaryNP) (lambda r (lambda np (lambda s (call @countSuperlative (var s) (string max) (call @reverse (var r)) (var np)))))) # Scmax
|
||||
# S5
|
||||
(rule $CP1 (that the least number of $UnaryNP $VP/NP) (lambda np (lambda r (lambda s (call @countSuperlative (var s) (string min) (call @reverse (var r)) (var np)))))) # Scvmin
|
||||
(rule $CP1 (that the most number of $UnaryNP $VP/NP) (lambda np (lambda r (lambda s (call @countSuperlative (var s) (string max) (call @reverse (var r)) (var np)))))) # Scvmax
|
||||
# C1
|
||||
(rule $CP1 (that has $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string =) (var num)))))) # Ceq
|
||||
(rule $CP1 (that has less than $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string <) (var num)))))) # Cl
|
||||
(rule $CP1 (that has more than $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string >) (var num)))))) # Cg
|
||||
(rule $CP1 (that has at most $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string <=) (var num)))))) # Cleq
|
||||
(rule $CP1 (that has at least $Num $RelNP) (lambda num (lambda r (lambda s (call @countComparative (var s) (var r) (string >=) (var num)))))) # Cgeq
|
||||
# C2
|
||||
(rule $CP1 (that $VP/NP $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string =) (var num) (var np))))))) # Ceq
|
||||
(rule $CP1 (that $VP/NP less than $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string <) (var num) (var np))))))) # Cl
|
||||
(rule $CP1 (that $VP/NP more than $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string >) (var num) (var np))))))) # Cg
|
||||
(rule $CP1 (that $VP/NP at most $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string <=) (var num) (var np))))))) # Cleq
|
||||
(rule $CP1 (that $VP/NP at least $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (var r) (string >=) (var num) (var np))))))) # Cgeq
|
||||
# C3
|
||||
(rule $CP1 (that is $RelNP of $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string =) (var num) (var np))))))) # Ceq
|
||||
(rule $CP1 (that is $RelNP of less than $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <) (var num) (var np))))))) # Cl
|
||||
(rule $CP1 (that is $RelNP of more than $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >) (var num) (var np))))))) # Cg
|
||||
(rule $CP1 (that is $RelNP of at most $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <=) (var num) (var np))))))) # Cleq
|
||||
(rule $CP1 (that is $RelNP of at least $Num $UnaryNP) (lambda r (lambda num (lambda np (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >=) (var num) (var np))))))) # Cgeq
|
||||
# C4
|
||||
(rule $CP1 (that $Num $UnaryNP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string =) (var num) (var np))))))) # Ceq
|
||||
(rule $CP1 (that less than $Num $UnaryNP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <) (var num) (var np))))))) # Cl
|
||||
(rule $CP1 (that more than $Num $UnaryNP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >) (var num) (var np))))))) # Cg
|
||||
(rule $CP1 (that at most $Num $UnaryNP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string <=) (var num) (var np))))))) # Cleq
|
||||
(rule $CP1 (that at least $Num $UnaryNP $VP/NP) (lambda num (lambda np (lambda r (lambda s (call @countComparative (var s) (call @reverse (var r)) (string >=) (var num) (var np))))))) # Cgeq
|
||||
)
|
||||
|
||||
# Regex
|
||||
(when (and parse regex)
|
||||
(rule $CP (that $VP/NP1 at least $Num $NP) (lambda v (lambda num (lambda n (call + (string "\(") ((var v) (var n)) (string "\)") (string "{") (var num) (string ",}"))))))
|
||||
(rule $CP (that $VP/NP1 at most $Num $NP) (lambda v (lambda num (lambda n (call + (string "\(") ((var v) (var n)) (string "\)") (string "{0,") (var num) (string "}"))))))
|
||||
(rule $CP (that $VP/NP1 $Num $NP) (lambda v (lambda num (lambda n (call + (string "\(") ((var v) (var n)) (string "\)") (string "{") (var num) (string "}"))))))
|
||||
(rule $CP (that $VP/NP at least $Num $NP) (lambda v (lambda num (lambda n ((var v) (call + (string "\(") (var n) (string "\)") (string "{") (var num) (string ",}")))))))
|
||||
(rule $CP (that $VP/NP at most $Num $NP) (lambda v (lambda num (lambda n ((var v) (call + (string "\(") (var n) (string "\)") (string "{0,") (var num) (string "}")))))))
|
||||
(rule $CP (that $VP/NP $Num $NP) (lambda v (lambda num (lambda n ((var v) (call + (string "\(") (var n) (string "\)") (string "{") (var num) (string "}")))))))
|
||||
)
|
||||
(when (and generate regex)
|
||||
(rule $CP1 (that $VP/NP1 at least $Num $UnaryNP) (lambda v (lambda num (lambda n (call + (string "\(") ((var v) (var n)) (string "\)") (string "{") (var num) (string ",}"))))))
|
||||
(rule $CP1 (that $VP/NP1 at most $Num $UnaryNP) (lambda v (lambda num (lambda n (call + (string "\(") ((var v) (var n)) (string "\)") (string "{0,") (var num) (string "}"))))))
|
||||
(rule $CP1 (that $VP/NP1 $Num $UnaryNP) (lambda v (lambda num (lambda n (call + (string "\(") ((var v) (var n)) (string "\)") (string "{") (var num) (string "}"))))))
|
||||
(rule $CP1 (that $VP/NP at least $Num $UnaryNP) (lambda v (lambda num (lambda n ((var v) (call + (string "\(") (var n) (string "\)") (string "{") (var num) (string ",}")))))))
|
||||
(rule $CP1 (that $VP/NP at most $Num $UnaryNP) (lambda v (lambda num (lambda n ((var v) (call + (string "\(") (var n) (string "\)") (string "{0,") (var num) (string "}")))))))
|
||||
(rule $CP1 (that $VP/NP $Num $UnaryNP) (lambda v (lambda num (lambda n ((var v) (call + (string "\(") (var n) (string "\)") (string "{") (var num) (string "}")))))))
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Construct NPs from CPs
|
||||
|
||||
(when (and parse general)
|
||||
# G3
|
||||
(rule $NPCP ($NP $CP) (JoinFn backward betaReduce))
|
||||
(rule $NPCP ($NPCP and $CP) (JoinFn backward betaReduce))
|
||||
(rule $NP ($NPCP) (IdentityFn))
|
||||
)
|
||||
(when (and generate general)
|
||||
# G3
|
||||
(rule $NP0 ($UnaryNP $CP00) (JoinFn backward betaReduce))
|
||||
(rule $NP1 ($UnaryNP $CP0) (JoinFn backward betaReduce))
|
||||
(rule $NP1 ($UnaryNP $CP1) (JoinFn backward betaReduce))
|
||||
(rule $NPCP1 ($UnaryNP $CP00) (JoinFn backward betaReduce))
|
||||
(rule $NP1 ($NPCP1 and $CP00) (JoinFn backward betaReduce))
|
||||
)
|
||||
(when geo880
|
||||
(rule $NP2 ($NP0 $CP00) (JoinFn backward betaReduce))
|
||||
(rule $NP2 ($NP0 $CP0) (JoinFn backward betaReduce))
|
||||
(rule $NP2 ($NP0 $CP1) (JoinFn backward betaReduce))
|
||||
(rule $NP2 ($UnaryNP $CP2) (JoinFn backward betaReduce))
|
||||
)
|
||||
(when geo440
|
||||
(rule $NP2 ($NP0 $CP00) (JoinFn backward betaReduce))
|
||||
(rule $NP2 ($NP0 $CP0) (JoinFn backward betaReduce))
|
||||
(rule $NP2 ($UnaryNP $CP2) (JoinFn backward betaReduce))
|
||||
)
|
||||
|
||||
# Regex
|
||||
(when (and parse regex)
|
||||
(rule $CP ($CP and $CP) (lambda c1 (lambda c2 (call + (string "\(\(") (var c1) (string "\)&\(") (var c2) (string "\)\)")))))
|
||||
(rule $NP ($NP $CP) (JoinFn betaReduce forward))
|
||||
)
|
||||
(when (and generate regex)
|
||||
(rule $CP1 ($CP00 and $CP00) (lambda c1 (lambda c2 (call + (string "\(\(") (var c1) (string "\)&\(") (var c2) (string "\)\)")))))
|
||||
(rule $NP1 ($UnaryNP $CP0) (JoinFn betaReduce forward))
|
||||
(rule $NP1 ($UnaryNP $CP1) (JoinFn betaReduce forward))
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Transformations
|
||||
|
||||
(when (and parse general)
|
||||
# T1
|
||||
(rule $NP ($RelNP of $NP) (lambda r (lambda s (call @getProperty (var s) (var r))))) # Tr
|
||||
# T3
|
||||
(rule $NP ($NP or $NP) (lambda n1 (lambda n2 (call @concat (var n1) (var n2))))) # Tdisj
|
||||
)
|
||||
(when (and generate general)
|
||||
# T1
|
||||
(rule $NP0 ($RelNP of $EntityNP1) (lambda r (lambda s (call @getProperty (var s) (var r))))) # Tr
|
||||
# T3
|
||||
(rule $NP0 ($EntityNP1 or $EntityNP2) (lambda n1 (lambda n2 (call @concat (var n1) (var n2))))) # Tdisj
|
||||
)
|
||||
(when (or geo880 geo440)
|
||||
(rule $NP2 ($RelNP of $NP0) (lambda r (lambda s (call @getProperty (var s) (var r))))) # Tr
|
||||
(rule $NP2 ($RelNP of $NP1) (lambda r (lambda s (call @getProperty (var s) (var r))))) # Tr
|
||||
)
|
||||
|
||||
(when (and parse regex)
|
||||
(rule $NP ($NP or $NP) (lambda n1 (lambda n2 (call + (string "\(") (var n1) (string "|") (var n2) (string "\)")))))
|
||||
)
|
||||
(when (and generate regex)
|
||||
# TODO - do we want "location of meeting"? I think not, probably only "area of california"??
|
||||
(rule $NP0 ($EntityNP1 or $EntityNP2) (lambda n1 (lambda n2 (call + (string "\(") (var n1) (string "|") (var n2) (string "\)")))))
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Transformations: events
|
||||
|
||||
(when parse
|
||||
# T2
|
||||
(rule $EventNP ($Rel0NP $NP) (lambda r0 (lambda e (call @getProperty (var e) (call @reverse (var r0)))))) # student John
|
||||
(rule $EventNPCP ($EventNP $CP) (JoinFn backward betaReduce)) # ... whose field of study is history
|
||||
(rule $EventNPCP ($EventNPCP and $CP) (JoinFn backward betaReduce))
|
||||
(rule $EventNP ($EventNPCP) (IdentityFn))
|
||||
(rule $NP ($RelNP of $EventNP) (lambda r (lambda x (call @getProperty (var x) (var r))))) # university of ...
|
||||
# T2'
|
||||
(rule $NP ($Rel0NP $CP) (lambda r (lambda cp (call @getProperty ((var cp) (call @domain (var r))) (var r))))) # student whose field of study is history
|
||||
)
|
||||
(when generate
|
||||
# T2
|
||||
(rule $EventNP0 ($Rel0NP $EntityNP1) (lambda r0 (lambda e (call @getProperty (var e) (call @reverse (var r0)))))) # student John
|
||||
(rule $EventNP1 ($EventNP0 $CP00) (JoinFn backward betaReduce)) # ... whose field of study is history
|
||||
(rule $NP0 ($RelNP of $EventNP0) (lambda r (lambda x (call @getProperty (var x) (var r))))) # university of ...
|
||||
(rule $NP1 ($RelNP of $EventNP1) (lambda r (lambda x (call @getProperty (var x) (var r))))) # university of ...
|
||||
# T2'
|
||||
(rule $NP0 ($Rel0NP $CP0) (lambda r (lambda cp (call @getProperty ((var cp) (call @domain (var r))) (var r))))) # student whose field of study is history
|
||||
(rule $NP1 ($Rel0NP $CP1) (lambda r (lambda cp (call @getProperty ((var cp) (call @domain (var r))) (var r))))) # student whose field of study is history
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Transformations: binary operators
|
||||
|
||||
(when parse
|
||||
# T4
|
||||
(rule $BinaryOpRight ($BinaryOp $NP) (JoinFn betaReduce forward))
|
||||
(rule $NP ($NP $BinaryOpRight) (JoinFn betaReduce backward))
|
||||
)
|
||||
(when generate
|
||||
# T4
|
||||
(rule $BinaryOpRight ($BinaryOp $EntityNP2) (JoinFn betaReduce forward))
|
||||
(rule $NP0 ($EntityNP1 $BinaryOpRight) (JoinFn betaReduce backward))
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Aggregation
|
||||
|
||||
(when (and parse general)
|
||||
# A1
|
||||
(rule $NP (number of $NP) (lambda x (call .size (var x)))) # An
|
||||
# A2
|
||||
(rule $NP (total $RelNP of $NP) (lambda r (lambda n (call @aggregate (string sum) (call @getProperty (var n) (var r)))))) # At
|
||||
# A3
|
||||
(rule $NP (average $RelNP of $NP) (lambda r (lambda n (call @aggregate (string avg) (call @getProperty (var n) (var r)))))) # Am
|
||||
# TODO: add 'all' before $NP's
|
||||
)
|
||||
(when (and generate general)
|
||||
# A1
|
||||
(rule $NP1 (number of $UnaryNP) (lambda x (call .size (var x)))) # An
|
||||
# A2
|
||||
(rule $NP1 (total $RelNP of $UnaryNP) (lambda r (lambda n (call @aggregate (string sum) (call @getProperty (var n) (var r)))))) # At
|
||||
# A3
|
||||
(rule $NP1 (average $RelNP of $UnaryNP) (lambda r (lambda n (call @aggregate (string avg) (call @getProperty (var n) (var r)))))) # Am
|
||||
)
|
||||
# Supporting more for geo880
|
||||
(when (or geo880 geo440)
|
||||
# A1
|
||||
(rule $NP2 (number of $NP0) (lambda x (call .size (var x)))) # An
|
||||
(rule $NP2 (number of $NP1) (lambda x (call .size (var x)))) # An
|
||||
# A2
|
||||
(rule $NP2 (total $RelNP of $NP0) (lambda r (lambda n (call @aggregate (string sum) (call @getProperty (var n) (var r)))))) # At
|
||||
(rule $NP2 (total $RelNP of $NP1) (lambda r (lambda n (call @aggregate (string sum) (call @getProperty (var n) (var r)))))) # At
|
||||
# A3
|
||||
(rule $NP2 (average $RelNP of $NP0) (lambda r (lambda n (call @aggregate (string avg) (call @getProperty (var n) (var r)))))) # Am
|
||||
(rule $NP2 (average $RelNP of $NP1) (lambda r (lambda n (call @aggregate (string avg) (call @getProperty (var n) (var r)))))) # Am
|
||||
)
|
||||
|
||||
############################################################
|
||||
# Top-level
|
||||
|
||||
(when (and parse general)
|
||||
(rule $ROOT ($NP) (lambda x (call @listValue (var x))))
|
||||
)
|
||||
(when (and generate general)
|
||||
(rule $ROOT ($NP0) (lambda x (call @listValue (var x))))
|
||||
(rule $ROOT ($NP1) (lambda x (call @listValue (var x))))
|
||||
)
|
||||
(when (or geo880 geo440)
|
||||
(rule $ROOT ($NP2) (lambda x (call @listValue (var x))))
|
||||
(rule $ROOT ($UnaryNP) (lambda x (call @listValue (var x))))
|
||||
)
|
||||
|
||||
(when (and parse regex)
|
||||
(rule $ROOT ($NP) (IdentityFn))
|
||||
)
|
||||
(when (and generate regex)
|
||||
(rule $ROOT ($NP0) (IdentityFn))
|
||||
(rule $ROOT ($NP1) (IdentityFn))
|
||||
)
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
# Unittest
|
||||
|
||||
# R1
|
||||
(example (utterance "state whose capital is sacramento"))
|
||||
(example (utterance "state whose capital is not sacramento"))
|
||||
(example (utterance "state whose capital is capital of california"))
|
||||
(example (utterance "mountain whose elevation is smaller than elevation of mount whitney"))
|
||||
(example (utterance "mountain whose elevation is larger than elevation of mount whitney"))
|
||||
(example (utterance "mountain whose elevation is at most elevation of mount whitney"))
|
||||
(example (utterance "mountain whose elevation is at least elevation of mount whitney"))
|
||||
(example (utterance "city that is in california and that is capital of california"))
|
||||
|
||||
# R2
|
||||
(example (utterance "city that is in california"))
|
||||
(example (utterance "city that not is in california"))
|
||||
# R3
|
||||
# R4
|
||||
(example (utterance "state that california borders"))
|
||||
(example (utterance "state that california not borders"))
|
||||
(example (utterance "river that traverses california"))
|
||||
# R5
|
||||
(example (utterance "city that is major"))
|
||||
|
||||
# S1
|
||||
(example (utterance "state that has the largest area"))
|
||||
(example (utterance "state that has the smallest area"))
|
||||
# S2
|
||||
# S3
|
||||
(example (utterance "state that borders the most number of state"))
|
||||
(example (utterance "state that borders the least number of state"))
|
||||
|
||||
# C1
|
||||
(example (utterance "river that is in two state"))
|
||||
(example (utterance "river that is in more than two state"))
|
||||
(example (utterance "river that is in less than two state"))
|
||||
(example (utterance "river that is in at most two state"))
|
||||
(example (utterance "river that is in at least two state"))
|
||||
|
||||
# T1
|
||||
(example (utterance "population of sacramento"))
|
||||
# T2: TODO
|
||||
# T3
|
||||
(example (utterance "california or texas"))
|
||||
# T4
|
||||
# A1
|
||||
(example (utterance "number of lake"))
|
||||
# A2
|
||||
(example (utterance "total elevation of mountain"))
|
||||
(example (utterance "average elevation of mountain"))
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (city) (ConstantFn fb:en.city))
|
||||
(rule $EntityNP1 (sacramento) (ConstantFn fb:en.city.sacramento_ca))
|
||||
(rule $EntityNP2 (austin) (ConstantFn fb:en.city.austin_tx))
|
||||
|
||||
(rule $TypeNP (state) (ConstantFn fb:en.state))
|
||||
(rule $EntityNP1 (california) (ConstantFn fb:en.state.california))
|
||||
(rule $EntityNP2 (texas) (ConstantFn fb:en.state.texas))
|
||||
|
||||
(rule $TypeNP (river) (ConstantFn fb:en.river))
|
||||
(rule $EntityNP1 (colorado river) (ConstantFn fb:en.river.colorado))
|
||||
(rule $EntityNP2 (red river) (ConstantFn fb:en.river.red))
|
||||
|
||||
(rule $TypeNP (lake) (ConstantFn fb:en.lake))
|
||||
(rule $EntityNP1 (lake tahoe) (ConstantFn fb:en.lake.tahoe))
|
||||
(rule $EntityNP2 (lake huron) (ConstantFn fb:en.lake.huron))
|
||||
|
||||
(rule $TypeNP (mountain) (ConstantFn fb:en.mountain))
|
||||
(rule $EntityNP1 (mount whitney) (ConstantFn fb:en.mountain.whitney))
|
||||
(rule $EntityNP2 (mount rainier) (ConstantFn fb:en.mountain.rainier))
|
||||
|
||||
(rule $TypeNP (place) (ConstantFn fb:en.place))
|
||||
(rule $EntityNP1 (death valley) (ConstantFn fb:en.place.death_valley))
|
||||
(rule $EntityNP2 (pacific ocean) (ConstantFn fb:en.place.pacific_ocean))
|
||||
|
||||
# Unaries
|
||||
(rule $VP (is major) (ConstantFn (string major_city)))
|
||||
(rule $VP (is major) (ConstantFn (string major_river)))
|
||||
(rule $VP (is major) (ConstantFn (string major_lake)))
|
||||
(rule $VP (is a capital) (ConstantFn (string capital_city)))
|
||||
|
||||
# Properties
|
||||
(rule $VP/NP (is contained by) (ConstantFn (string loc_city_state)))
|
||||
(rule $VP/NP (is contained by) (ConstantFn (string loc_lake_state)))
|
||||
(rule $VP/NP (is contained by) (ConstantFn (string loc_mountain_state)))
|
||||
(rule $VP/NP (is contained by) (ConstantFn (string loc_state_country)))
|
||||
(rule $VP/NP (is contained by) (ConstantFn (string loc_place_state)))
|
||||
|
||||
(rule $VP/NP (traverses) (ConstantFn (string traverse_river_state)))
|
||||
(rule $VP/NP (borders) (ConstantFn (string next_to_state_state)))
|
||||
|
||||
(rule $RelNP (capital) (ConstantFn (string capital_state_city)))
|
||||
(rule $RelNP (area) (ConstantFn (string area_state_length^2)))
|
||||
(rule $RelNP (area) (ConstantFn (string area_city_length^2)))
|
||||
(rule $RelNP (area) (ConstantFn (string area_country_length^2)))
|
||||
(rule $RelNP (area) (ConstantFn (string area_lake_length^2)))
|
||||
(rule $RelNP (length) (ConstantFn (string len_river_length)))
|
||||
(rule $RelNP (elevation) (ConstantFn (string elevation_mountain_length)))
|
||||
(rule $RelNP (elevation) (ConstantFn (string elevation_place_length)))
|
||||
(rule $RelNP (population) (ConstantFn (string population_city_count)))
|
||||
(rule $RelNP (population) (ConstantFn (string population_state_count)))
|
||||
(rule $RelNP (population) (ConstantFn (string population_country_count)))
|
||||
(rule $RelNP (density) (ConstantFn (string density_state_count)))
|
||||
(rule $RelNP (density) (ConstantFn (string density_city_count)))
|
||||
(rule $RelNP (density) (ConstantFn (string density_country_count)))
|
||||
|
||||
# simple lexicon
|
||||
(rule $EntityNP1 ($PHRASE) (SimpleLexiconFn))
|
||||
(rule $EntityNP2 ($PHRASE) (SimpleLexiconFn))
|
||||
|
||||
# TODO: named is a funny relation: do we really want to include it?
|
||||
# I estimate less than 2% requires the 'name' property, so let's punt
|
||||
#(rule $EntityNP1 (sacramento) (ConstantFn (string Sacramento)))
|
||||
#(rule $EntityNP1 (california) (ConstantFn (string California)))
|
||||
#(rule $EntityNP1 (carson river) (ConstantFn (string Carson River)))
|
||||
#(rule $EntityNP1 (lake austin) (ConstantFn (string Lake Austin)))
|
||||
#(rule $EntityNP1 (mount whitney) (ConstantFn (string Mount Whitney)))
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
# R1
|
||||
(example (utterance "housing unit whose neighborhood is chelsea"))
|
||||
(example (utterance "housing unit whose housing type is apartment"))
|
||||
|
||||
# R2
|
||||
(example (utterance "housing unit that allows cats"))
|
||||
(example (utterance "housing unit that allows dogs"))
|
||||
|
||||
# R3
|
||||
(example (utterance "housing unit whose size is at least 800 square feet"))
|
||||
|
||||
(example (utterance "number of housing type"))
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Agile grammar for housing (based on craigslist)
|
||||
# - What is the price of house?
|
||||
# - How many apartments are there?
|
||||
# - Create house with price 225000
|
||||
# - Move price of house from 225000 to 250000
|
||||
# - Is there any apartment whose rent is less than 2000
|
||||
|
||||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (housing unit) (ConstantFn en.housing_unit))
|
||||
(rule $EntityNP1 (123 sesame street) (ConstantFn en.housing_unit.123_sesame_street))
|
||||
(rule $EntityNP2 (900 mission ave) (ConstantFn en.housing_unit.900_mission_ave))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (monthly rent) (ConstantFn (string rent)))
|
||||
(rule $EntityNP1 (1500 dollars) (ConstantFn (number 1500 en.dollar)))
|
||||
(rule $EntityNP2 (2000 dollars) (ConstantFn (number 2000 en.dollar)))
|
||||
|
||||
(rule $RelNP (size) (ConstantFn (string size)))
|
||||
(rule $EntityNP1 (800 square feet) (ConstantFn (number 800 en.square_feet)))
|
||||
(rule $EntityNP2 (1000 square feet) (ConstantFn (number 1000 en.square_feet)))
|
||||
|
||||
(rule $RelNP (posting date) (ConstantFn (string posting_date)))
|
||||
(rule $EntityNP1 (jan 2) (ConstantFn (date 2015 1 2)))
|
||||
(rule $EntityNP2 (feb 3) (ConstantFn (date 2015 2 3)))
|
||||
|
||||
(rule $RelNP (neighborhood) (ConstantFn (string neighborhood)))
|
||||
(rule $TypeNP (neighborhood) (ConstantFn en.neighborhood))
|
||||
(rule $EntityNP1 (midtown west) (ConstantFn en.neighborhood.midtown_west))
|
||||
(rule $EntityNP2 (chelsea) (ConstantFn en.neighborhood.chelsea))
|
||||
|
||||
(rule $RelNP (housing type) (ConstantFn (string housing_type)))
|
||||
(rule $TypeNP (housing type) (ConstantFn en.housing))
|
||||
(rule $EntityNP1 (apartment) (ConstantFn en.housing.apartment))
|
||||
(rule $EntityNP2 (condo) (ConstantFn en.housing.condo))
|
||||
|
||||
# Unaries
|
||||
(rule $VP (allows cats) (ConstantFn (string allows_cats)))
|
||||
(rule $VP (allows dogs) (ConstantFn (string allows_dogs)))
|
||||
(rule $VP (has a private bath) (ConstantFn (string has_private_bath)))
|
||||
(rule $VP (has a private room) (ConstantFn (string has_private_room)))
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# Used to trigger generation (without looking at utterance).
|
||||
(example (utterance null))
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
# Agile grammar for publications
|
||||
|
||||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (article) (ConstantFn en.article))
|
||||
(rule $EntityNP1 (multivariate data analysis) (ConstantFn en.article.multivariate_data_analysis))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (author) (ConstantFn (string author)))
|
||||
(rule $TypeNP (person) (ConstantFn en.person))
|
||||
(rule $EntityNP1 (efron) (ConstantFn en.person.efron))
|
||||
(rule $EntityNP2 (lakoff) (ConstantFn en.person.lakoff))
|
||||
|
||||
(rule $RelNP (venue) (ConstantFn (string venue)))
|
||||
(rule $TypeNP (venue) (ConstantFn en.venue))
|
||||
(rule $EntityNP1 (annals of statistics) (ConstantFn en.venue.annals_of_statistics))
|
||||
(rule $EntityNP2 (computational linguistics) (ConstantFn en.venue.computational_linguistics))
|
||||
|
||||
(rule $RelNP (publication date) (ConstantFn (string publication_date)))
|
||||
(rule $EntityNP1 (2004) (ConstantFn (date 2004 -1 -1)))
|
||||
(rule $EntityNP2 (2010) (ConstantFn (date 2010 -1 -1)))
|
||||
|
||||
(rule $VP/NP (cites) (ConstantFn (string cites)))
|
||||
|
||||
# Unaries
|
||||
(rule $VP (won an award) (ConstantFn (string won_award)))
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
# Agile grammar for recipes (based on allrecipes.com)
|
||||
|
||||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (recipe) (ConstantFn en.recipe))
|
||||
(rule $EntityNP1 (rice pudding) (ConstantFn en.recipe.rice_pudding))
|
||||
(rule $EntityNP2 (quiche) (ConstantFn en.recipe.quiche))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (preparation time) (ConstantFn (string preparation_time)))
|
||||
(rule $RelNP (cooking time) (ConstantFn (string cooking_time)))
|
||||
(rule $Value (10) (ConstantFn (number 10 en.minute)))
|
||||
(rule $Value (30) (ConstantFn (number 300 en.minute)))
|
||||
|
||||
(rule $RelNP (cuisine) (ConstantFn (string cuisine)))
|
||||
(rule $TypeNP (cuisine) (ConstantFn fb:en.cuisine))
|
||||
(rule $EntityNP1 (chinese) (ConstantFn (string chinese)))
|
||||
(rule $EntityNP2 (french) (ConstantFn (string french)))
|
||||
|
||||
(rule $VP/NP (requires) (ConstantFn (string requires)))
|
||||
(rule $TypeNP (ingredient) (ConstantFn en.ingredient))
|
||||
(rule $EntityNP1 (milk) (ConstantFn en.ingredient.milk))
|
||||
(rule $EntityNP2 (spinach) (ConstantFn en.ingredient.spinach))
|
||||
|
||||
(rule $VP/NP (is for) (ConstantFn (string meal)))
|
||||
(rule $TypeNP (meal) (ConstantFn en.meal))
|
||||
(rule $EntityNP1 (lunch) (ConstantFn en.meal.lunch))
|
||||
(rule $EntityNP2 (dinner) (ConstantFn en.meal.dinner))
|
||||
|
||||
(rule $RelNP (posting date) (ConstantFn (string posting_date)))
|
||||
(rule $EntityNP1 (2004) (ConstantFn (date 2004 -1 -1)))
|
||||
(rule $EntityNP2 (2010) (ConstantFn (date 2010 -1 -1)))
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# Unittest
|
||||
|
||||
# Type
|
||||
(example (utterance "restaurant"))
|
||||
|
||||
# R1
|
||||
(example (utterance "restaurant that takes reservations"))
|
||||
(example (utterance "restaurant that has outdoor seating"))
|
||||
|
||||
# R2
|
||||
(example (utterance "restaurant whose cuisine is chinese"))
|
||||
(example (utterance "restaurant whose neighborhood is tribeca"))
|
||||
|
||||
# R3
|
||||
(example (utterance "restaurant whose price rating is 2 dollar signs"))
|
||||
(example (utterance "restaurant whose star rating is at least 3 stars"))
|
||||
|
||||
# R4
|
||||
(example (utterance "restaurant that serves lunch"))
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# Agile grammar for restaurants (based on yelp)
|
||||
|
||||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (restaurant) (ConstantFn en.restaurant))
|
||||
(rule $EntityNP1 (thai cafe) (ConstantFn en.restaurant.thai_cafe))
|
||||
(rule $EntityNP2 (pizzeria juno) (ConstantFn en.restaurant.pizzeria_juno))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (star rating) (ConstantFn (string star_rating)))
|
||||
(rule $EntityNP1 (3 stars) (ConstantFn (number 3 en.star)))
|
||||
(rule $EntityNP2 (5 stars) (ConstantFn (number 5 en.star)))
|
||||
|
||||
(rule $RelNP (price rating) (ConstantFn (string price_rating)))
|
||||
(rule $EntityNP1 (2 dollar signs) (ConstantFn (number 2 en.dollar_sign)))
|
||||
(rule $EntityNP2 (3 dollar signs) (ConstantFn (number 3 en.dollar_sign)))
|
||||
|
||||
(rule $RelNP (number of reviews) (ConstantFn (string reviews)))
|
||||
(rule $EntityNP1 (30 reviews) (ConstantFn (number 30 en.review)))
|
||||
(rule $EntityNP2 (40 reviews) (ConstantFn (number 40 en.review)))
|
||||
|
||||
(rule $RelNP (neighborhood) (ConstantFn (string neighborhood)))
|
||||
(rule $TypeNP (neighborhood) (ConstantFn en.neighborhood))
|
||||
(rule $EntityNP1 (midtown west) (ConstantFn en.neighborhood.midtown_west))
|
||||
(rule $EntityNP2 (chelsea) (ConstantFn en.neighborhood.chelsea))
|
||||
|
||||
(rule $RelNP (cuisine) (ConstantFn (string cuisine)))
|
||||
(rule $TypeNP (cuisine) (ConstantFn en.cuisine))
|
||||
(rule $EntityNP1 (thai) (ConstantFn en.cuisine.thai))
|
||||
(rule $EntityNP2 (italian) (ConstantFn en.cuisine.italian))
|
||||
|
||||
(rule $VP/NP (serves) (ConstantFn (string meals)))
|
||||
(rule $TypeNP (meal) (ConstantFn en.food))
|
||||
(rule $EntityNP1 (lunch) (ConstantFn en.food.lunch))
|
||||
(rule $EntityNP2 (dinner) (ConstantFn en.food.dinner))
|
||||
|
||||
# Unaries
|
||||
(rule $VP (takes reservations) (ConstantFn (string reserve)))
|
||||
(rule $VP (takes credit cards) (ConstantFn (string credit)))
|
||||
(rule $VP (has outdoor seating) (ConstantFn (string outdoor)))
|
||||
(rule $VP (has take-out) (ConstantFn (string takeout)))
|
||||
(rule $VP (has delivery) (ConstantFn (string delivery)))
|
||||
(rule $VP (has waiter service) (ConstantFn (string waiter)))
|
||||
(rule $VP (is good for kids) (ConstantFn (string kids)))
|
||||
(rule $VP (is good for groups) (ConstantFn (string groups)))
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
# Unittest
|
||||
|
||||
# G1
|
||||
(example (utterance "google"))
|
||||
# G2
|
||||
(example (utterance "gender"))
|
||||
# G3
|
||||
(example (utterance "person whose gender is male and that is friends with alice"))
|
||||
|
||||
# R1
|
||||
(example (utterance "person whose gender is male"))
|
||||
(example (utterance "person whose gender is not male"))
|
||||
(example (utterance "person whose birthdate is smaller than 2010"))
|
||||
(example (utterance "person whose birthdate is larger than 2010"))
|
||||
(example (utterance "person whose birthdate is at most 2010"))
|
||||
(example (utterance "person whose birthdate is at least 2010"))
|
||||
(example (utterance "person whose height is at least 200 cm"))
|
||||
# R2
|
||||
(example (utterance "person that is friends with alice"))
|
||||
(example (utterance "person that not is friends with alice"))
|
||||
# R3
|
||||
(example (utterance "city that is birthplace of alice"))
|
||||
(example (utterance "city that is not birthplace of alice"))
|
||||
# R4
|
||||
(example (utterance "person that is friends with alice"))
|
||||
(example (utterance "person that not is friends with alice"))
|
||||
# R5
|
||||
(example (utterance "person that is logged in"))
|
||||
|
||||
# S1
|
||||
(example (utterance "person that has the smallest birthdate"))
|
||||
(example (utterance "person that has the largest birthdate"))
|
||||
# S2
|
||||
(example (utterance "person that has the most birthplace"))
|
||||
(example (utterance "person that has the least birthplace"))
|
||||
# S3
|
||||
(example (utterance "person that is friends with the most person"))
|
||||
(example (utterance "person that is friends with the least person"))
|
||||
|
||||
# C1
|
||||
(example (utterance "person that is friends with 1 person"))
|
||||
(example (utterance "person that is friends with less than 1 person"))
|
||||
(example (utterance "person that is friends with more than 1 person"))
|
||||
(example (utterance "person that is friends with at most 1 person"))
|
||||
(example (utterance "person that is friends with at least 1 person"))
|
||||
|
||||
# T1
|
||||
(example (utterance "birthdate of alice"))
|
||||
# T2
|
||||
(example (utterance "employer of employee bob"))
|
||||
(example (utterance "employer of employee bob whose start date is 2004"))
|
||||
# T3
|
||||
(example (utterance "alice or bob"))
|
||||
# T4 (none)
|
||||
|
||||
# A1
|
||||
(example (utterance "number of person"))
|
||||
# A2
|
||||
(example (utterance "total height of person"))
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# Agile grammar for social network (think Facebook Graph Search)
|
||||
|
||||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (person) (ConstantFn en.person))
|
||||
(rule $EntityNP1 (alice) (ConstantFn en.person.alice))
|
||||
(rule $EntityNP2 (bob) (ConstantFn en.person.bob))
|
||||
|
||||
(rule $VP/NP (is friends with) (ConstantFn (string friend)))
|
||||
|
||||
# Properties
|
||||
(rule $RelNP (birthdate) (ConstantFn (string birthdate)))
|
||||
(rule $EntityNP1 (2004) (ConstantFn (date 2004 -1 -1)))
|
||||
(rule $EntityNP2 (2010) (ConstantFn (date 2010 -1 -1)))
|
||||
|
||||
(rule $RelNP (gender) (ConstantFn (string gender)))
|
||||
(rule $TypeNP (gender) (ConstantFn en.gender))
|
||||
(rule $EntityNP1 (male) (ConstantFn en.gender.male))
|
||||
(rule $EntityNP2 (female) (ConstantFn en.gender.female))
|
||||
|
||||
(rule $RelNP (relationship status) (ConstantFn (string relationship_status)))
|
||||
(rule $TypeNP (relationship status) (ConstantFn en.relationship_status))
|
||||
(rule $EntityNP1 (single) (ConstantFn en.relationship_status.single))
|
||||
(rule $EntityNP2 (married) (ConstantFn en.relationship_status.married))
|
||||
|
||||
(rule $RelNP (birthplace) (ConstantFn (string birthplace)))
|
||||
(rule $TypeNP (city) (ConstantFn en.city))
|
||||
(rule $EntityNP1 (new york) (ConstantFn en.city.new_york))
|
||||
(rule $EntityNP2 (beijing) (ConstantFn en.city.bejing))
|
||||
|
||||
(rule $RelNP (height) (ConstantFn (string height)))
|
||||
(rule $EntityNP1 (180 cm) (ConstantFn (number 180 en.cm)))
|
||||
(rule $EntityNP2 (200 cm) (ConstantFn (number 200 en.cm)))
|
||||
|
||||
# Education
|
||||
#(rule $EventNP (education) (ConstantFn (call @getProperty (call @singleton en.education) (string !type)))) # for debugging
|
||||
|
||||
(rule $Rel0NP (student) (ConstantFn (string student)))
|
||||
|
||||
(rule $RelNP (university) (ConstantFn (string university)))
|
||||
(rule $TypeNP (university) (ConstantFn en.university))
|
||||
(rule $EntityNP1 (brown university) (ConstantFn en.university.brown))
|
||||
(rule $EntityNP2 (ucla) (ConstantFn en.university.ucla))
|
||||
|
||||
(rule $RelNP (field of study) (ConstantFn (string field_of_study)))
|
||||
(rule $TypeNP (field) (ConstantFn en.field))
|
||||
(rule $EntityNP1 (computer science) (ConstantFn en.field.computer_science))
|
||||
(rule $EntityNP2 (history) (ConstantFn en.field.history))
|
||||
|
||||
(rule $RelNP (start date) (ConstantFn (string education_start_date)))
|
||||
(rule $RelNP (end date) (ConstantFn (string education_end_date)))
|
||||
|
||||
# Employment
|
||||
|
||||
(rule $Rel0NP (employee) (ConstantFn (string employee)))
|
||||
|
||||
(rule $RelNP (employer) (ConstantFn (string employer)))
|
||||
(rule $TypeNP (company) (ConstantFn en.company))
|
||||
(rule $EntityNP1 (mckinsey) (ConstantFn en.company.mckinsey))
|
||||
(rule $EntityNP2 (google) (ConstantFn en.company.google))
|
||||
|
||||
(rule $RelNP (job title) (ConstantFn (string job_title)))
|
||||
(rule $TypeNP (job title) (ConstantFn en.job_title))
|
||||
(rule $EntityNP1 (software engineer) (ConstantFn en.job_title.software_engineer))
|
||||
(rule $EntityNP2 (program manager) (ConstantFn en.job_title.program_manager))
|
||||
|
||||
(rule $RelNP (start date) (ConstantFn (string employment_start_date)))
|
||||
(rule $RelNP (end date) (ConstantFn (string employment_end_date)))
|
||||
|
||||
# Unaries
|
||||
(rule $VP (is logged in) (ConstantFn (string logged_in)))
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
############################################################
|
||||
# Domain general
|
||||
# Keep track of coarse types:
|
||||
# - Entity: "day 1"
|
||||
# - Value: "3pm"
|
||||
# - EntitySet: "meeting"
|
||||
# - ValueSet: "start time of meeting"
|
||||
|
||||
(def @allEntities edu.stanford.nlp.sempre.SimpleWorld.allEntities)
|
||||
(def @filter edu.stanford.nlp.sempre.SimpleWorld.filter)
|
||||
(def @merge edu.stanford.nlp.sempre.SimpleWorld.merge)
|
||||
(def @getProperty edu.stanford.nlp.sempre.SimpleWorld.getProperty)
|
||||
(def @superlative edu.stanford.nlp.sempre.SimpleWorld.superlative)
|
||||
(def @sum edu.stanford.nlp.sempre.SimpleWorld.sum)
|
||||
(def @list edu.stanford.nlp.sempre.SimpleWorld.list)
|
||||
(def @act edu.stanford.nlp.sempre.SimpleWorld.act)
|
||||
|
||||
# Operations
|
||||
(def @create edu.stanford.nlp.sempre.SimpleWorld.create)
|
||||
(def @remove edu.stanford.nlp.sempre.SimpleWorld.remove)
|
||||
(def @change edu.stanford.nlp.sempre.SimpleWorld.change)
|
||||
|
||||
# General values
|
||||
(rule $Value ($PHRASE) (NumberFn) (anchored 1))
|
||||
(rule $Value ($PHRASE) (DateFn) (anchored 1))
|
||||
|
||||
(rule $TypeSet ($Type) (lambda t (call @filter (call @allEntities) (string type) (string =) (var t))))
|
||||
|
||||
# 0. Show me all meetings
|
||||
(rule $Command (show me all $TypeSet) (IdentityFn))
|
||||
|
||||
# 1. Get property: "what is the location of meeting"
|
||||
(rule $Command (what is the $EntityProperty of the $TypeSet) (lambda p (lambda s (call @getProperty (var s) (var p)))))
|
||||
(rule $Command (what is the $ValueProperty of the $TypeSet) (lambda p (lambda s (call @getProperty (var s) (var p)))))
|
||||
# what is the block left of block whose color is red
|
||||
(rule $Command (what is the $TypeSet $BinaryProperty the $TypeSet) (lambda s1 (lambda p (lambda s2 (call @getProperty (var s2) (var p))))))
|
||||
|
||||
# 2. Filter: "what is the meeting whose location is cafe"
|
||||
(rule $Command (what is the $TypeSet whose $EntityProperty is $Entity) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string =) (var v))))))
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is $Value) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string =) (var v))))))
|
||||
(rule $Command (what is the $TypeSet whose $EntityProperty is not $Entity) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string !=) (var v))))))
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is not $Value) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string !=) (var v))))))
|
||||
|
||||
# 3. Count: what is the number of meeting whose location is cafe
|
||||
(rule $Command (what is the number of $TypeSet whose $EntityProperty is $Entity) (lambda s (lambda p (lambda v (call .size (call @filter (var s) (var p) (string =) (var v)))))))
|
||||
|
||||
# 4. Sum: what is the total length of meeting
|
||||
(rule $Command (what is the total $ValueProperty of all $TypeSet) (lambda p (lambda s (call @sum (call @getProperty (var s) (var p))))))
|
||||
|
||||
# 5. Comparatives: what is the meeting whose start time is smaller than 10am
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is smaller than $Value) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string <) (var v))))))
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is larger than $Value) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string >) (var v))))))
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is at most $Value) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string <=) (var v))))))
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is at least $Value) (lambda s (lambda p (lambda v (call @filter (var s) (var p) (string >=) (var v))))))
|
||||
|
||||
# 6. Superlatives - what is the meeting with the smallest start time
|
||||
(rule $Command (what is the $TypeSet with the smallest $ValueProperty) (lambda s (lambda p (call @superlative (var s) (string min) (var p)))))
|
||||
(rule $Command (what is the $TypeSet with the largest $ValueProperty) (lambda s (lambda p (call @superlative (var s) (string max) (var p)))))
|
||||
|
||||
# 7. create and remove
|
||||
(rule $Command (create a $Type with $EntityProperty $Entity) (lambda t (lambda p (lambda v (call @create (var t) (var p) (var v))))))
|
||||
(rule $Command (create a $Type with $ValueProperty $Value) (lambda t (lambda p (lambda v (call @create (var t) (var p) (var v))))))
|
||||
(rule $Command (remove the $TypeSet with $EntityProperty $Entity) (lambda s (lambda p (lambda v (call @remove (call @filter (var s) (var p) (string =) (var v)))))))
|
||||
|
||||
# 8. Two joins
|
||||
(rule $Command (what is the $TypeSet whose $ValueProperty is $ValueProperty of $TypeSet) (lambda s1 (lambda p1 (lambda p2 (lambda s2 (call @filter (var s1) (var p1) (string =) (call @getProperty (var s2) (var p2))))))))
|
||||
|
||||
(rule $ROOT ($Command) (lambda x (call .toString (var x))))
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
friend fb:en.person.alice fb:en.person.bob
|
||||
enemy fb:en.person.alice fb:en.person.eve
|
||||
person fb:en.person.alice
|
||||
person fb:en.person.bob
|
||||
person fb:en.person.eve
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
(include general.grammar)
|
||||
|
||||
# Types
|
||||
(rule $TypeNP (person) (ConstantFn en.person))
|
||||
|
|
@ -143,6 +143,52 @@ addModule('fullfreebase-types', 'Freebase types', lambda {
|
|||
# You need to unzip these yourself and move these files to the right place.
|
||||
})
|
||||
|
||||
addModule('tables', 'Semantic parsing with execution on tables', lambda {
|
||||
# CSV reader
|
||||
pull('/u/nlp/data/semparse/resources/opencsv-3.0.jar')
|
||||
})
|
||||
|
||||
addModule('tables-data', 'Data for semantic parsing with execution on tables', lambda {
|
||||
pull('/u/nlp/data/semparse/wikitable/data', 'data/tables/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/wikitable/mturk-trivia-data/csv', 'data/tables/', {:symlink => true})
|
||||
})
|
||||
|
||||
|
||||
addModule('overnight', 'Creating a parser for multiple domains', lambda {
|
||||
# Geo evaluation
|
||||
pull('/u/nlp/data/semparse/overnight/geo880.db', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/geo880/geo880-train.examples', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/geo880/geo880-test.examples', 'data/overnight/', {:symlink => true})
|
||||
|
||||
|
||||
# Cache for turking
|
||||
pull('/u/nlp/data/semparse/overnight/cache/', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/layouts/', 'data/overnight/', {:symlink => true})
|
||||
|
||||
# Pull testing code
|
||||
pull('/u/nlp/data/semparse/overnight/test/', 'data/overnight/', {:symlink => true})
|
||||
|
||||
# Pull geo880
|
||||
pull('/u/nlp/data/semparse/overnight/geo880/geo880.paraphrases.train.superlatives.examples', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/geo880/geo880.paraphrases.train.superlatives2.examples', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/geo880/geo880.lexicon', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/geo880/geo880.predicate.dict', 'data/overnight/', {:symlink => true})
|
||||
|
||||
# Pull dependencies for everything else
|
||||
domains = ['geo880', 'regex', 'publications', 'socialnetwork', 'restaurants', 'blocks', 'calendar', 'housing', 'basketball', 'recipes', 'calendarplus']
|
||||
domains.each do |domain|
|
||||
pull('/u/nlp/data/semparse/overnight/' + domain + '/' + domain + '.paraphrases.train.examples', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/' + domain + '/' + domain + '.paraphrases.test.examples', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/' + domain + '/' + domain + '.paraphrases.groups', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/' + domain + '/' + domain + '.word_alignments.berkeley', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/' + domain + '/' + domain + '.phrase_alignments', 'data/overnight/', {:symlink => true})
|
||||
pull('/u/nlp/data/semparse/overnight/' + domain + '/' + domain + '-ppdb.txt', 'data/overnight/', {:symlink => true})
|
||||
end
|
||||
|
||||
# Pull the independent sets for calendar
|
||||
pull('/u/nlp/data/semparse/overnight/calendar/eval/calendar.test.turk.examples', 'data/overnight/', {:symlink => true})
|
||||
})
|
||||
|
||||
|
||||
############################################################
|
||||
|
||||
|
|
|
|||
440
run
440
run
|
|
@ -13,8 +13,21 @@ def addMode(name, description, func)
|
|||
$modes << [name, description, func]
|
||||
end
|
||||
|
||||
def header(modules='core')
|
||||
def codalab(dependencies=nil)
|
||||
# Set @cl=1 to run job on CodaLab
|
||||
dependencies ||= l(':fig', ':lib', ':module-classes.txt', ':libsempre')
|
||||
l(
|
||||
letDefault(:cl, 0),
|
||||
sel(:cl,
|
||||
l(),
|
||||
l('cl', 'run', dependencies, '---'),
|
||||
nil),
|
||||
nil)
|
||||
end
|
||||
|
||||
def header(modules='core', codalabDependencies=nil)
|
||||
l(
|
||||
codalab(codalabDependencies),
|
||||
# Queuing system
|
||||
letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
|
||||
# Create execution directory
|
||||
|
|
@ -23,7 +36,18 @@ def header(modules='core')
|
|||
'java',
|
||||
'-ea',
|
||||
'-Dmodules='+modules,
|
||||
'-Xmx10g',
|
||||
# Memory size
|
||||
letDefault(:memsize, 'default'),
|
||||
sel(:memsize, {
|
||||
'tiny' => l('-Xms2G', '-Xmx4G'),
|
||||
'low' => l('-Xms5G', '-Xmx7G'),
|
||||
'default' => l('-Xms8G', '-Xmx10G'),
|
||||
'medium' => l('-Xms12G', '-Xmx14G'),
|
||||
'high' => l('-Xms20G', '-Xmx24G'),
|
||||
'higher' => l('-Xms40G', '-Xmx50G'),
|
||||
'impressive' => l('-Xms75G', '-Xmx90G'),
|
||||
}),
|
||||
# Classpath
|
||||
'-cp', 'libsempre/*:lib/*',
|
||||
# Profiling
|
||||
letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'),
|
||||
|
|
@ -46,7 +70,11 @@ def figOpts; l(o('execDir', '_OUTPATH_'), o('overwriteExecDir'), o('addToView',
|
|||
|
||||
addMode('test', 'Run unit tests', lambda { |e|
|
||||
l(
|
||||
'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*', 'org.testng.TestNG',
|
||||
'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*',
|
||||
lambda { |e|
|
||||
e.key?(:sparqlserver) ? "-Dsparqlserver=http://#{e[:sparqlserver]}/sparql" : l()
|
||||
},
|
||||
'org.testng.TestNG',
|
||||
lambda { |e|
|
||||
if e[:class]
|
||||
l('-testclass', 'edu.stanford.nlp.sempre.' + e[:class])
|
||||
|
|
@ -131,6 +159,7 @@ def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath)
|
|||
when 'local' then l( # Use files directly - don't run more than one job that does this!
|
||||
o('Lexicon.cachePath', 'LexiconFn.cache'),
|
||||
o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'),
|
||||
o('FreebaseSearch.cachePath', 'FreebaseSearch.cache'),
|
||||
nil)
|
||||
else l(
|
||||
o('Lexicon.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+lexiconFnCachePath),
|
||||
|
|
@ -225,7 +254,8 @@ def webquestions
|
|||
nil),
|
||||
|
||||
# Grammar
|
||||
o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'),
|
||||
letDefault(:grammar, 1),
|
||||
sel(:grammar, l(), l(o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'))),
|
||||
|
||||
o('Parser.beamSize', 200), # {07/03/13}: WebQuestions is too slow to run with default 500, so set to 200 for now...
|
||||
|
||||
|
|
@ -350,9 +380,19 @@ nil) })
|
|||
addMode('sparqlserver', '(2) Start the SPARQL server [do this every time]', lambda { |e| l(
|
||||
scrOptions,
|
||||
required(:exec),
|
||||
'freebase/scripts/virtuoso', 'start',
|
||||
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
|
||||
lambda{|e| 3000+e[:exec]}, # port
|
||||
sel(nil,
|
||||
l(
|
||||
'freebase/scripts/virtuoso', 'start',
|
||||
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
|
||||
lambda{|e| 3000+e[:exec]}, # port
|
||||
nil),
|
||||
# Give everyone permissions so that anyone can kill the server if needed.
|
||||
l(
|
||||
'chmod', '-R', 'og=u',
|
||||
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
|
||||
nil),
|
||||
# To stop the server: freebase/scripts/virtuoso stop 3093
|
||||
nil),
|
||||
nil) })
|
||||
|
||||
# (3) Index the filtered RDF dump [takes 48 hours]
|
||||
|
|
@ -387,6 +427,7 @@ addMode('convertfree917', 'Convert the Free917 dataset', lambda { |e| l(
|
|||
nil) })
|
||||
|
||||
addMode('query', 'Query a single logical form or SPARQL', lambda { |e| l(
|
||||
codalab,
|
||||
'java', '-ea',
|
||||
'-cp', 'libsempre/*:lib/*',
|
||||
'edu.stanford.nlp.sempre.freebase.SparqlExecutor',
|
||||
|
|
@ -395,17 +436,18 @@ nil) })
|
|||
|
||||
############################################################
|
||||
|
||||
|
||||
# Just start a simple interactive shell to try out SEMPRE commands
|
||||
addMode('simple', 'Simple shell', lambda { |e| l(
|
||||
rlwrap, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
|
||||
o('interactive'),
|
||||
codalab, rlwrap, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
|
||||
o('Main.interactive'),
|
||||
nil) })
|
||||
|
||||
addMode('simple-sparql', 'Simple shell for querying SPARQL', lambda { |e| l(
|
||||
rlwrap, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
|
||||
codalab, rlwrap, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
|
||||
o('executor', 'freebase.SparqlExecutor'),
|
||||
sparqlOpts,
|
||||
o('interactive'),
|
||||
o('Main.interactive'),
|
||||
nil) })
|
||||
|
||||
addMode('simple-freebase', 'Simple shell for using Freebase', lambda { |e| l(
|
||||
|
|
@ -435,6 +477,382 @@ addMode('simple-freebase', 'Simple shell for using Freebase', lambda { |e| l(
|
|||
nil) })
|
||||
|
||||
|
||||
############################################################
|
||||
# {2014-12-27} [Percy]: Overnight semantic parsing
|
||||
addMode('overnight', 'Overnight semantic parsing', lambda { |e| l(
|
||||
'rlwrap',
|
||||
header('core,freebase,overnight'),
|
||||
'edu.stanford.nlp.sempre.Main',
|
||||
figOpts,
|
||||
o('JavaExecutor.convertNumberValues', false),
|
||||
o('useAnchorsOnce', true),
|
||||
o('trackLocalChoices'),
|
||||
o('JoinFn.typeInference', true),
|
||||
o('Builder.parser', 'FloatingParser'),
|
||||
o('FloatingParser.executeAllDerivations', 'true'),
|
||||
o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'),
|
||||
o('Learner.maxTrainIters', 1),
|
||||
#o('printAllPredictions'),
|
||||
o('Derivation.showUtterance'),
|
||||
letDefault(:domain, 'publications'),
|
||||
letDefault(:debug, 0),
|
||||
|
||||
selo(1, 'maxExamples', 'train:10', 'train:MAX'),
|
||||
|
||||
# Exact matching is needed on most simple domains
|
||||
# o('executor', 'FormulaMatchExecutor'),
|
||||
# o('Builder.valueEvaluator', 'ExactValueEvaluator'),
|
||||
|
||||
# Features
|
||||
o('FeatureExtractor.featureDomains', 'denotation'), # denotation features from general feature extractor
|
||||
o('FeatureExtractor.featureComputers', 'overnight.OvernightFeatureComputer'), #
|
||||
o('OvernightFeatureComputer.featureDomains', *overnightFeatureDomains),
|
||||
#o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
|
||||
# 'paraphrase :: skip-bigram,0.8', 'paraphrase :: skip-ppdb,0.2','denotation :: error,-1000'),
|
||||
o('coarsePrune'),
|
||||
o('OvernightDerivationPruningComputer.applyHardConstraints'),
|
||||
sel(2,
|
||||
l(), # no reg
|
||||
l(o('Params.l1Reg','lazy'),o('Params.l1RegCoeff',0)),
|
||||
l(o('Params.l1Reg','lazy'),o('Params.l1RegCoeff',0.001)),
|
||||
nil),
|
||||
# Set up the domain
|
||||
required(:domain),
|
||||
o('Grammar.inPaths', lambda { |e| 'overnight/' + e[:domain] + '.grammar' }),
|
||||
o('SimpleWorld.domain', lambda { |e| e[:domain] }),
|
||||
o('PPDBModel.ppdbModelPath', lambda { |e| 'lib/data/overnight/' + e[:domain] + '-ppdb.txt' }),
|
||||
#o('PPDBModel.ppdbModelPath', lambda { |e| 'overnightData/' + e[:domain] + '-ppdb.txt' }), # codalab
|
||||
o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2),
|
||||
o('FloatingParser.maxDepth', 11),
|
||||
o('Parser.beamSize', 20),
|
||||
letDefault(:alignment,1),
|
||||
sel(:alignment,
|
||||
o('wordAlignmentPath', 'lib/data/overnight/' + e[:domain] + '.word_alignments.heuristic'),
|
||||
o('wordAlignmentPath', 'lib/data/overnight/' + e[:domain] + '.word_alignments.berkeley'),
|
||||
#o('wordAlignmentPath', 'overnightData/' + e[:domain] + '.word_alignments.heuristic'), # codalab
|
||||
#o('wordAlignmentPath', 'overnightData/' + e[:domain] + '.word_alignments.berkeley'),
|
||||
nil),
|
||||
o('phraseAlignmentPath', 'lib/data/overnight/' + e[:domain] + '.phrase_alignments'),
|
||||
o('PPDBModel.ppdbModelPath', 'lib/data/overnight/' + e[:domain] + '-ppdb.txt'),
|
||||
#o('phraseAlignmentPath', 'overnightData/' + e[:domain] + '.phrase_alignments'), # codalab
|
||||
#o('PPDBModel.ppdbModelPath', 'overnightData/' + e[:domain] + '-ppdb.txt'),
|
||||
o('DerivationPruner.pruningComputers', ['overnight.OvernightDerivationPruningComputer']),
|
||||
o('Dataset.inPaths',
|
||||
'train:lib/data/overnight/' + e[:domain] + '.paraphrases.train.examples',
|
||||
'test:lib/data/overnight/' + e[:domain] + '.paraphrases.test.examples'),
|
||||
#'train:overnightData/' + e[:domain] + '.paraphrases.train.examples', # codalab
|
||||
#'test:overnightData/' + e[:domain] + '.paraphrases.test.examples'),
|
||||
sel(:domain, {
|
||||
'regex' => l(
|
||||
letDefault(:data,0),
|
||||
sel(:data,
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples')),
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples', 'test:lib/data/overnight/regex.examples.train.json')),
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples', 'test:lib/data/overnight/regex.examples.test.json')),
|
||||
nil),
|
||||
# selo(:debug, 'Dataset.inPaths', 'train:lib/data/overnight/regex.paraphrases.train.examples', 'train:regex/regex-debug.examples'),
|
||||
o('executor', 'JavaExecutor'),
|
||||
o('Builder.valueEvaluator', 'regex.RegexValueEvaluator'),
|
||||
o('OvernightFeatureComputer.approxEmptyDenotation', 'true'),
|
||||
# o('OvernightFeatureComputer.featureDomains', *regexFeatureDomains),
|
||||
o('JavaExecutor.convertNumberValues', true),
|
||||
o('Grammar.tags','generate','regex'),
|
||||
o('lowerCaseTokens', false), # case sensitivity is important for correctness of regular expressions
|
||||
o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
|
||||
'paraphrase :: skip-bigram,0.8', 'paraphrase :: skip-ppdb,0.2','denotation :: error,-1000'),
|
||||
nil),
|
||||
'geo880' => l(
|
||||
letDefault(:data,0),
|
||||
sel(:data,
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives.examples')),
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives.examples', 'test:lib/data/overnight/geo880-train.examples')),
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives2.examples', 'test:lib/data/overnight/geo880-train.examples')),
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives.examples', 'test:lib/data/overnight/geo880-test.examples')),
|
||||
l(o('Dataset.inPaths', 'train:lib/data/overnight/geo880.paraphrases.train.superlatives2.examples', 'test:lib/data/overnight/geo880-test.examples')),
|
||||
nil),
|
||||
o('Parser.beamSize', 20),
|
||||
o('initialization', 'paraphrase :: match,1', 'paraphrase :: size,-0.1', 'paraphrase :: ppdb,0.3',
|
||||
'lf :: edu.stanford.nlp.sempre.SimpleWorld.superlative& superlative,10',
|
||||
'root :: pos0=WRB&returnType=class edu.stanford.nlp.sempre.NumberValue,10'),
|
||||
o('FloatingParser.maxDepth', 11),
|
||||
o('Grammar.tags','generate','general', 'geo880'),
|
||||
o('SimpleLexicon.inPaths', 'lib/data/overnight/geo880.lexicon'),
|
||||
#o('Params.l1Reg','lazy'),
|
||||
#o('Params.l1RegCoeff',0.00001),
|
||||
o('OvernightDerivationPruningComputer.usePredicateDict'),
|
||||
o('OvernightDerivationPruningComputer.predicateDictPath','lib/data/overnight/geo880.predicate.dict'),
|
||||
nil),
|
||||
'calendar' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'calendarplus' => l(
|
||||
o('Grammar.tags','generate','general','geo440'),
|
||||
o('Grammar.inPaths','overnight/calendar.grammar'),
|
||||
o('SimpleWorld.domain', 'calendar'),
|
||||
nil),
|
||||
'blocks' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'restaurants' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'housing' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'socialnetwork' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'publications' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'basketball' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
'recipes' => l(
|
||||
o('Grammar.tags','generate','general'),
|
||||
nil),
|
||||
}),
|
||||
nil) })
|
||||
|
||||
def overnightFeatureDomains
|
||||
[
|
||||
'match',
|
||||
'ppdb',
|
||||
'skip-bigram',
|
||||
'root',
|
||||
'alignment',
|
||||
'lexical',
|
||||
'root_lexical',
|
||||
'lf',
|
||||
'simpleworld',
|
||||
nil].compact
|
||||
end
|
||||
|
||||
############################################################
|
||||
# {5/27/15} [Ice]
|
||||
addMode('tables', 'QA on HTML tables', lambda { |e| l(
|
||||
# Add @cldir=1 to use CodaLab's directory paths
|
||||
letDefault(:cldir, 0),
|
||||
# Usual header
|
||||
rlwrap, header('core,tables,corenlp'),
|
||||
# Select class
|
||||
letDefault(:class, 'main'),
|
||||
sel(:class, {
|
||||
'main' => 'edu.stanford.nlp.sempre.Main',
|
||||
'check' => 'edu.stanford.nlp.sempre.tables.test.DPParserChecker',
|
||||
'align' => 'edu.stanford.nlp.sempre.tables.alignment.IBMAligner',
|
||||
'dump' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedDumper',
|
||||
'load' => 'edu.stanford.nlp.sempre.tables.serialize.SerializedLoader',
|
||||
'stats' => 'edu.stanford.nlp.sempre.tables.test.TableStatsComputer',
|
||||
}),
|
||||
# Fig parameters
|
||||
selo(:cldir, 'execDir', '_OUTPATH_', '.'),
|
||||
o('overwriteExecDir'), o('addToView', 0), o('jarFiles', 'libsempre/*'),
|
||||
sel(:cldir, l(), '>/dev/null'),
|
||||
# Set environment for table execution
|
||||
o('executor', 'tables.lambdadcs.LambdaDCSExecutor'),
|
||||
o('Builder.valueEvaluator', 'tables.TableValueEvaluator'),
|
||||
o('TargetValuePreprocessor.targetValuePreprocessor', 'tables.TableValuePreprocessor'),
|
||||
o('NumberFn.unitless'), o('NumberFn.alsoTestByConversion'),
|
||||
o('TypeInference.typeLookup', 'tables.TableTypeLookup'),
|
||||
o('JoinFn.specializedTypeCheck', false), o('JoinFn.typeInference', true),
|
||||
# Parser
|
||||
letDefault(:parser, 'floatsize'),
|
||||
sel(:parser, {
|
||||
'floatsize' => l(
|
||||
o('Builder.parser', 'FloatingParser'),
|
||||
o('useSizeInsteadOfDepth'),
|
||||
o('FloatingParser.maxDepth', 15),
|
||||
nil),
|
||||
'baseline' => l(
|
||||
o('Builder.parser', 'tables.baseline.TableBaselineParser'),
|
||||
nil),
|
||||
'dummy' => o('Builder.parser', 'tables.serialize.DummyParser'),
|
||||
}),
|
||||
o('FloatingParser.useAnchorsOnce', true),
|
||||
letDefault(:pruning, 1),
|
||||
sel(:pruning,
|
||||
l(),
|
||||
l(
|
||||
o('DerivationPruner.pruningStrategies', *tablesPruningStrategies),
|
||||
o('DerivationPruner.pruningComputers', 'tables.TableDerivationPruningComputer'),
|
||||
nil),
|
||||
nil),
|
||||
# Grammar
|
||||
tablesGrammarPaths,
|
||||
# Dataset
|
||||
letDefault(:data, 'none'),
|
||||
letDefault(:unseen, 0),
|
||||
tablesDataPaths,
|
||||
# Verbosity
|
||||
o('FeatureVector.ignoreZeroWeight'),
|
||||
o('maxPrintedPredictions', 10), o('maxPrintedTrue', 10), o('logFeaturesLimit', 10),
|
||||
letDefault(:verbose, 0),
|
||||
sel(:verbose,
|
||||
l(),
|
||||
l(
|
||||
o('showRules'),
|
||||
o('Parser.verbose', 2),
|
||||
o('JoinFn.verbose', 3),
|
||||
o('JoinFn.showTypeCheckFailures'),
|
||||
nil),
|
||||
nil),
|
||||
# Language Analyzer
|
||||
letDefault(:lang, 'corenlp'),
|
||||
sel(:lang, {
|
||||
'simple' => o('LanguageAnalyzer', 'SimpleAnalyzer'),
|
||||
'corenlp' => l(o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'), o('annotators', *'tokenize ssplit pos lemma ner'.split)),
|
||||
'fullcorenlp' => l(o('LanguageAnalyzer', 'corenlp.CoreNLPAnalyzer'), o('annotators', *'tokenize ssplit pos lemma ner parse'.split)),
|
||||
}),
|
||||
# Training
|
||||
letDefault(:train, 0),
|
||||
sel(:train,
|
||||
l(),
|
||||
l(
|
||||
o('combineFromFloatingParser'),
|
||||
sel(:unseen, unbalancedTrainDevSplit, l()),
|
||||
o('maxTrainIters', 3),
|
||||
o('showValues', false), o('showFirstValue'),
|
||||
nil),
|
||||
l(
|
||||
# for dumping derivations (@class=dump)
|
||||
# force unbalancedTrainDevSplit + combine from floating parser
|
||||
o('combineFromFloatingParser'), o('DPParser.cheat'),
|
||||
sel(:unseen, unbalancedTrainDevSplit, l()),
|
||||
nil),
|
||||
nil),
|
||||
# Regularization
|
||||
letDefault(:l1, 1),
|
||||
sel(:l1,
|
||||
l(),
|
||||
l(o('Params.l1Reg','lazy'), o('Params.l1RegCoeff', '3e-5')),
|
||||
l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0, 0.00001, 0.0001, 0.001, 0.01)),
|
||||
l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0001, 0.0003)),
|
||||
l(o('Params.l1Reg','lazy'), selo(nil, 'Params.l1RegCoeff', 0.00001, 0.00003, 0.0005)),
|
||||
nil),
|
||||
# Features
|
||||
letDefault(:feat, 'none'),
|
||||
sel(:feat, {
|
||||
'none' => l(), # No features (random)
|
||||
'some' => l( # Add your own features! (only set up the feature computers)
|
||||
o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
|
||||
nil),
|
||||
'all' => l( # All features
|
||||
o('FeatureExtractor.featureDomains', 'custom-denotation phrase-predicate phrase-denotation headword-denotation missing-predicate'.split),
|
||||
o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
|
||||
nil),
|
||||
'baseline' => l( # For the baseline classifier
|
||||
o('FeatureExtractor.featureDomains', 'custom-denotation phrase-denotation headword-denotation table-baseline'.split),
|
||||
o('FeatureExtractor.featureComputers', 'tables.baseline.TableBaselineFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
|
||||
nil),
|
||||
'ablate' => l(
|
||||
o('FeatureExtractor.featureComputers', 'tables.features.PhrasePredicateFeatureComputer tables.features.PhraseDenotationFeatureComputer'.split),
|
||||
selo(nil,
|
||||
'FeatureExtractor.featureDomains',
|
||||
'phrase-predicate phrase-denotation headword-denotation missing-predicate'.split,
|
||||
'custom-denotation phrase-denotation headword-denotation missing-predicate'.split,
|
||||
'custom-denotation phrase-predicate headword-denotation missing-predicate'.split,
|
||||
'custom-denotation phrase-predicate phrase-denotation missing-predicate'.split,
|
||||
'custom-denotation phrase-predicate phrase-denotation headword-denotation'.split,
|
||||
nil),
|
||||
nil),
|
||||
}),
|
||||
letDefault(:featOp, 'careful'),
|
||||
sel(:featOp, {
|
||||
'none' => l(),
|
||||
'base' => l(
|
||||
o('usePredicateLemma'), o('usePhraseLemmaOnly'),
|
||||
nil),
|
||||
'careful' => l(
|
||||
o('usePredicateLemma'), o('usePhraseLemmaOnly'),
|
||||
o('maxNforLexicalizeAllPairs', 2),
|
||||
o('traverseWithFormulaTypes'), o('reverseNameValueConversion', 'allBang'),
|
||||
o('lookUnderCellProperty'), o('useGenericCellType'),
|
||||
o('computeFuzzyMatchPredicates'),
|
||||
nil),
|
||||
}),
|
||||
nil) })
|
||||
|
||||
def tablesGrammarPaths
|
||||
lambda { |e|
|
||||
baseDir = ['tables/grammars/', 'grammars/'][e[:cldir]]
|
||||
l(
|
||||
letDefault(:grammar, 'combined-all'),
|
||||
sel(:grammar, {
|
||||
'restrict' => o('Grammar.inPaths', "#{baseDir}restrict.grammar"),
|
||||
'simple' => o('Grammar.inPaths', "#{baseDir}simple.grammar"),
|
||||
'combined' => o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
'combined-jnc' => l( # WQ baseline
|
||||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'movement count'.split),
|
||||
nil),
|
||||
'combined-cut' => l( # No intersection / union
|
||||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'movement comparison count aggregate superlative arithmetic'.split),
|
||||
nil),
|
||||
'combined-all' => l( # Default
|
||||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'alternative movement comparison count aggregate superlative arithmetic merge'.split),
|
||||
nil),
|
||||
'combined-trigger' => l( # Use trigger words for operations
|
||||
o('Grammar.inPaths', "#{baseDir}combined.grammar"),
|
||||
o('Grammar.tags', *'t-alternative t-movement t-comparison t-count t-aggregate t-superlative t-arithmetic merge'.split),
|
||||
nil),
|
||||
}),
|
||||
nil)
|
||||
}
|
||||
end
|
||||
|
||||
def tablesDataPaths
|
||||
lambda { |e|
|
||||
baseDir = ['lib/data/tables/data/', 'WikiTableQuestions/data/'][e[:cldir]]
|
||||
csvDir = ['lib/data/tables/', 'WikiTableQuestions/'][e[:cldir]]
|
||||
datasets = {
|
||||
'none' => l(),
|
||||
# Pristine test test
|
||||
'test' => l(
|
||||
o('Dataset.inPaths',
|
||||
"train,#{baseDir}training.examples",
|
||||
"test,#{baseDir}pristine-unseen-tables.examples"),
|
||||
o('Dataset.trainFrac', 1), o('Dataset.devFrac', 0),
|
||||
let(:unseen, 1),
|
||||
nil),
|
||||
}
|
||||
# Development sets: 80:20 random split of training data
|
||||
['1', '2', '3', '4', '5'].each do |x|
|
||||
datasets['u-' + x] = l(
|
||||
o('Dataset.inPaths',
|
||||
"train,#{baseDir}random-split-seed-#{x}-train.examples",
|
||||
"dev,#{baseDir}random-split-seed-#{x}-test.examples",
|
||||
nil),
|
||||
let(:unseen, 1),
|
||||
nil)
|
||||
end
|
||||
# That's it!
|
||||
l(
|
||||
o('TableKnowledgeGraph.baseCSVDir', csvDir),
|
||||
sel(:data, datasets),
|
||||
nil)
|
||||
}
|
||||
end
|
||||
|
||||
def tablesPruningStrategies
|
||||
[
|
||||
# Formula
|
||||
"singleton",
|
||||
"multipleSuperlatives",
|
||||
"sameMerge",
|
||||
"forwardBackward",
|
||||
"doubleNext",
|
||||
# Denotation
|
||||
"emptyDenotation",
|
||||
"nonLambdaError",
|
||||
"tooManyValues",
|
||||
"badSuperlativeHead",
|
||||
"mistypedMerge",
|
||||
nil].compact
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,4 +7,5 @@ else
|
|||
fi
|
||||
|
||||
d=`dirname $0`
|
||||
java -cp $d/../lib/checkstyle/checkstyle-6.1.1-all.jar com.puppycrawl.tools.checkstyle.Main -c `dirname $0`/checkstyle.xml $files
|
||||
prog="$d/../lib/checkstyle/checkstyle-6.6-all.jar"
|
||||
java -cp $prog com.puppycrawl.tools.checkstyle.Main -c `dirname $0`/checkstyle.xml $files
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@
|
|||
<property name="basedir" value="${basedir}"/>
|
||||
-->
|
||||
|
||||
<property name="fileExtensions" value="java, properties, xml"/>
|
||||
|
||||
<!-- Checks that a package-info.java file exists for each package. -->
|
||||
<!-- See http://checkstyle.sf.net/config_javadoc.html#JavadocPackage -->
|
||||
<!-- RELAX -->
|
||||
|
|
@ -76,8 +78,16 @@
|
|||
<!-- <property name="fileExtensions" value="java"/> -->
|
||||
<!-- </module> -->
|
||||
|
||||
<!-- RELAX: Allow warning suppression -->
|
||||
<!-- Usage: Add @SuppressWarnings({"checkname"}) -->
|
||||
<!-- checkname must be lowercased -->
|
||||
<module name="SuppressWarningsFilter" />
|
||||
|
||||
<module name="TreeWalker">
|
||||
|
||||
<!-- RELAX: Allow warning suppression -->
|
||||
<module name="SuppressWarningsHolder" />
|
||||
|
||||
<!-- Checks for Javadoc comments. -->
|
||||
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
|
||||
<!-- RELAX -->
|
||||
|
|
@ -167,9 +177,6 @@
|
|||
<!-- RELAX: allow numbers -->
|
||||
<!--<module name="MagicNumber"/>-->
|
||||
<module name="MissingSwitchDefault"/>
|
||||
<module name="RedundantThrows">
|
||||
<property name="suppressLoadErrors" value="true"/>
|
||||
</module>
|
||||
<module name="SimplifyBooleanExpression"/>
|
||||
<module name="SimplifyBooleanReturn"/>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
class LexicalEntry:
|
||||
def __init__(self, l, f, t):
|
||||
self.lexeme=l.strip()
|
||||
self.formula=f.strip()
|
||||
self.type=t.strip()
|
||||
|
||||
out = open(sys.argv[2],'w')
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
for line in f:
|
||||
tokens = line.split("\t")
|
||||
if len(tokens) > 2:
|
||||
continue
|
||||
if(tokens[0] == "loc_city"):
|
||||
index = tokens[1].rfind('.')
|
||||
citystate = tokens[1][index+1:]
|
||||
city = citystate[0:citystate.rfind('_')]
|
||||
city = city.replace('_',' ').strip()
|
||||
entry = LexicalEntry(city, tokens[1], "fb:en.city")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
elif (tokens[0] == "loc_state"):
|
||||
index = tokens[1].rfind('.')
|
||||
state = tokens[1][index+1:].strip()
|
||||
state = state.replace('_',' ').strip()
|
||||
entry = LexicalEntry(state, tokens[1], "fb:en.state")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
elif tokens[0] == "loc_river":
|
||||
index = tokens[1].rfind('.')
|
||||
river = tokens[1][index+1:].strip()
|
||||
river = river.replace('_',' ').strip()
|
||||
entry = LexicalEntry(river+" river", tokens[1], "fb:en.river")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
elif (tokens[0] == "loc_place"):
|
||||
index = tokens[1].rfind('.')
|
||||
place = tokens[1][index+1:].strip()
|
||||
place = place.replace('_',' ').strip()
|
||||
entry = LexicalEntry(place, tokens[1], "fb:en.place")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
elif (tokens[0] == "loc_lake"):
|
||||
index = tokens[1].rfind('.')
|
||||
lake = tokens[1][index+1:].strip()
|
||||
lake = lake.replace('_',' ').strip()
|
||||
if not 'lake' in lake:
|
||||
lake = lake + " lake"
|
||||
entry = LexicalEntry(lake, tokens[1], "fb:en.lake")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
elif (tokens[0] == "loc_mountain"):
|
||||
index = tokens[1].rfind('.')
|
||||
mountain = tokens[1][index+1:].strip()
|
||||
mountain = mountain.replace('_',' ').strip()
|
||||
entry = LexicalEntry("mount " + mountain, tokens[1], "fb:en.mountain")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
elif (tokens[0] == "loc_country"):
|
||||
index = tokens[1].rfind('.')
|
||||
country = tokens[1][index+1:].strip()
|
||||
country = country.replace('_',' ').strip()
|
||||
entry = LexicalEntry(country, tokens[1], "fb:en.country")
|
||||
out.write(json.dumps(entry.__dict__)+'\n')
|
||||
|
||||
|
||||
out.close()
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
# Official evaluation script used to evaluate Freebase question answering
|
||||
# systems. Used for EMNLP 2013, ACL 2014 papers, etc.
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("Usage: %s <generated_file> <filtered_file>" % sys.argv[0])
|
||||
|
||||
generated = set()
|
||||
with open(sys.argv[1]) as f:
|
||||
for line in f:
|
||||
generated.add(line)
|
||||
|
||||
out = open(sys.argv[2],'w')
|
||||
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
for line in f:
|
||||
index = line.find(" not")
|
||||
if index != -1:
|
||||
newStr = line[0:index] + line[index+4:len(line)]
|
||||
if newStr in generated:
|
||||
out.write(line)
|
||||
else:
|
||||
out.write(line)
|
||||
out.close()
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/sh
|
||||
#Generate predictions file for evaluation script
|
||||
#arguments: (1) execution numebr (2) iteration number (3) final output file
|
||||
|
||||
fig/bin/tab e/$1.exec/learner.events iter group utterance targetValue predValue | grep -P "$2\ttest" | cut -f3,4,5 > pred_temp
|
||||
java -cp "libsempre/*:lib/*" edu.stanford.nlp.sempre.freebase.utils.FileUtils pred_temp $3
|
||||
rm pred_temp
|
||||
|
|
@ -20,6 +20,14 @@ pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}')
|
|||
if [ -n "$pid" ]; then kill $pid; fi
|
||||
ssh -N -n -L $port:$host:$port jacob.stanford.edu &
|
||||
|
||||
# Sparql server for geo
|
||||
host=jonsson
|
||||
port=3094
|
||||
echo "Tunnel localhost:$port => $host:$port"
|
||||
pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}')
|
||||
if [ -n "$pid" ]; then kill $pid; fi
|
||||
ssh -N -n -L $port:$host:$port jacob.stanford.edu &
|
||||
|
||||
# Sparql server for Paleo
|
||||
host=jonsson
|
||||
port=3021
|
||||
|
|
|
|||
|
|
@ -92,8 +92,8 @@ while true
|
|||
log("Testing...")
|
||||
run('git log -3', true) or restart # Print out last commit messages
|
||||
run('./pull-dependencies', true) or restart
|
||||
run('make clean', true) or restart
|
||||
run('make', true) or restart
|
||||
run('ant clean', true) or restart
|
||||
run('ant', true) or restart
|
||||
|
||||
run('scripts/find-hard-coded-paths.rb', true) or restart
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ public class AggregateFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof AggregateFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ public class ArithmeticFormula extends Formula {
|
|||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof ArithmeticFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ public class Builder {
|
|||
} catch (ClassNotFoundException e1) {
|
||||
throw new RuntimeException("Illegal parser: " + opts.parser);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Error while instantiating parser: " + opts.parser + "\n" + e);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ public class CallFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof CallFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -13,10 +13,12 @@ public final class CanonicalNames {
|
|||
private CanonicalNames() { }
|
||||
|
||||
// Standard type names
|
||||
public static final String PREFIX = "fb:";
|
||||
public static final String BOOLEAN = "fb:type.boolean";
|
||||
public static final String INT = "fb:type.int";
|
||||
public static final String FLOAT = "fb:type.float";
|
||||
public static final String DATE = "fb:type.datetime";
|
||||
public static final String TIME = "fb:type.time";
|
||||
public static final String TEXT = "fb:type.text";
|
||||
public static final String NUMBER = "fb:type.number";
|
||||
public static final String ENTITY = "fb:common.topic";
|
||||
|
|
@ -29,6 +31,24 @@ public final class CanonicalNames {
|
|||
public static final String TYPE = "fb:type.object.type";
|
||||
public static final String NAME = "fb:type.object.name";
|
||||
|
||||
// Unary: fb:domain.type [contains exactly one period]
|
||||
// Binary: fb:domain.type.property, <, >, etc.
|
||||
public static boolean isUnary(String s) {
|
||||
int i = s.indexOf('.');
|
||||
if (i == -1) return false;
|
||||
i = s.indexOf('.', i + 1);
|
||||
if (i == -1) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isBinary(String s) {
|
||||
int i = s.indexOf('.');
|
||||
if (i == -1) return false;
|
||||
i = s.indexOf('.', i + 1);
|
||||
if (i == -1) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Return whether |property| is the name of a reverse property.
|
||||
// Convention: ! is the prefix for reverses.
|
||||
public static boolean isReverseProperty(String property) {
|
||||
|
|
@ -39,4 +59,5 @@ public final class CanonicalNames {
|
|||
else return "!" + property;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,6 +77,8 @@ public abstract class ChartParserState extends ParserState {
|
|||
void addToChart(Derivation deriv) {
|
||||
if (parser.verbose(3)) LogInfo.logs("addToChart %s: %s", deriv.cat, deriv);
|
||||
|
||||
if (Parser.opts.pruneErrorValues && deriv.value instanceof ErrorValue) return;
|
||||
|
||||
List<Derivation> derivations = chart[deriv.start][deriv.end].get(deriv.cat);
|
||||
if (chart[deriv.start][deriv.end].get(deriv.cat) == null)
|
||||
chart[deriv.start][deriv.end].put(deriv.cat, derivations = new ArrayList<>());
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ public class Dataset {
|
|||
public double devFrac = 0;
|
||||
@Option(gloss = "Used to randomly divide training examples")
|
||||
public Random splitRandom = new Random(1);
|
||||
@Option(gloss = "whether to split dev from train")
|
||||
public boolean splitDevFromTrain = true;
|
||||
|
||||
@Option(gloss = "Only keep examples which have at most this number of tokens")
|
||||
public int maxTokens = Integer.MAX_VALUE;
|
||||
|
|
@ -44,8 +46,8 @@ public class Dataset {
|
|||
private LinkedHashMap<String, List<Example>> allExamples = new LinkedHashMap<String, List<Example>>();
|
||||
|
||||
// General statistics about the examples.
|
||||
private HashSet<String> tokenTypes = new HashSet<String>();
|
||||
private StatFig numTokensFig = new StatFig(); // For each example, number of tokens
|
||||
private final HashSet<String> tokenTypes = new HashSet<String>();
|
||||
private final StatFig numTokensFig = new StatFig(); // For each example, number of tokens
|
||||
|
||||
public Set<String> groups() { return allExamples.keySet(); }
|
||||
public List<Example> examples(String group) { return allExamples.get(group); }
|
||||
|
|
@ -123,7 +125,7 @@ public class Dataset {
|
|||
allExamples.put(groupInfo.group, examples = new ArrayList<Example>());
|
||||
readHelper(groupInfo.examples, maxExamples, examples, groupInfo.path);
|
||||
}
|
||||
splitDevFromTrain();
|
||||
if (opts.splitDevFromTrain) splitDevFromTrain();
|
||||
collectStats();
|
||||
|
||||
LogInfo.end_track();
|
||||
|
|
@ -140,8 +142,16 @@ public class Dataset {
|
|||
List<Example> trainExamples = new ArrayList<Example>();
|
||||
allExamples.put("train", trainExamples);
|
||||
List<Example> devExamples = allExamples.get("dev");
|
||||
if (devExamples == null)
|
||||
allExamples.put("dev", devExamples = new ArrayList<Example>());
|
||||
if (devExamples == null) {
|
||||
// Preserve order
|
||||
LinkedHashMap<String, List<Example>> newAllExamples = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, List<Example>> entry : allExamples.entrySet()) {
|
||||
newAllExamples.put(entry.getKey(), entry.getValue());
|
||||
if (entry.getKey().equals("train"))
|
||||
newAllExamples.put("dev", devExamples = new ArrayList<>());
|
||||
}
|
||||
allExamples = newAllExamples;
|
||||
}
|
||||
for (int i = 0; i < split1; i++)
|
||||
trainExamples.add(origTrainExamples.get(perm[i]));
|
||||
for (int i = split2; i < origTrainExamples.size(); i++)
|
||||
|
|
@ -165,7 +175,7 @@ public class Dataset {
|
|||
ex = new Example.Builder().withExample(ex).setId(id).createExample();
|
||||
}
|
||||
i++;
|
||||
ex.preprocess(LanguageAnalyzer.getSingleton());
|
||||
ex.preprocess();
|
||||
|
||||
// Skip example if too long
|
||||
if (ex.numTokens() > opts.maxTokens) continue;
|
||||
|
|
@ -190,7 +200,7 @@ public class Dataset {
|
|||
allExamples.put(group, examples = new ArrayList<Example>());
|
||||
readLispTreeHelper(path, maxExamples, examples);
|
||||
}
|
||||
splitDevFromTrain();
|
||||
if (opts.splitDevFromTrain) splitDevFromTrain();
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
|
|
@ -208,7 +218,7 @@ public class Dataset {
|
|||
|
||||
Example ex = Example.fromLispTree(tree, path + ":" + n); // Specify a default id if it doesn't exist
|
||||
n++;
|
||||
ex.preprocess(LanguageAnalyzer.getSingleton());
|
||||
ex.preprocess();
|
||||
|
||||
// Skip example if too long
|
||||
if (ex.numTokens() > opts.maxTokens) continue;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
public boolean showRules = false;
|
||||
@Option(gloss = "When printing derivations, to show canonical utterance")
|
||||
public boolean showUtterance = false;
|
||||
@Option(gloss = "When executing, show formulae (for debugging)")
|
||||
public boolean showExecutions = false;
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
|
@ -34,7 +36,11 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
public final String cat;
|
||||
public final int start;
|
||||
public final int end;
|
||||
|
||||
// Floating cell information
|
||||
// TODO(yushi): make fields final
|
||||
public String canonicalUtterance;
|
||||
private boolean[] anchoredTokens; // Tokens which anchored rules are defined on
|
||||
|
||||
// If this derivation is composed of other derivations
|
||||
public final Rule rule; // Which rule was used to produce this derivation? Set to nullRule if not.
|
||||
|
|
@ -58,6 +64,14 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
private final FeatureVector localFeatureVector; // Features
|
||||
double score = Double.NaN; // Weighted combination of features
|
||||
|
||||
// Used during parsing (by FeatureExtractor, SemanticFn) to cache arbitrary
|
||||
// computation across different sub-Derivations.
|
||||
// Convention:
|
||||
// - use the featureDomain, FeatureComputer or SemanticFn as the key.
|
||||
// - the value is whatever the FeatureExtractor needs.
|
||||
// This information should be set to null after parsing is done.
|
||||
private Map<String, Object> tempState;
|
||||
|
||||
// What the formula evaluates to (optionally set later; only non-null for the root Derivation)
|
||||
public Value value;
|
||||
public Evaluation executorStats;
|
||||
|
|
@ -141,12 +155,14 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
public Derivation createDerivation() {
|
||||
return new Derivation(
|
||||
cat, start, end, rule, children, formula, type,
|
||||
localFeatureVector, score, value, executorStats, compatibility, prob, canonicalUtterance);
|
||||
localFeatureVector, score, value, executorStats, compatibility, prob,
|
||||
canonicalUtterance);
|
||||
}
|
||||
}
|
||||
|
||||
Derivation(String cat, int start, int end, Rule rule, List<Derivation> children, Formula formula, SemType type,
|
||||
FeatureVector localFeatureVector, double score, Value value, Evaluation executorStats, double compatibility, double prob, String canonicalUtterance) {
|
||||
FeatureVector localFeatureVector, double score, Value value, Evaluation executorStats, double compatibility, double prob,
|
||||
String canonicalUtterance) {
|
||||
this.cat = cat;
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
|
|
@ -180,6 +196,7 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
public boolean containsIndex(int i) { return i < end && i >= start; }
|
||||
public Rule getRule() { return rule; }
|
||||
public Evaluation getExecutorStats() { return executorStats; }
|
||||
public FeatureVector getLocalFeatureVector() { return localFeatureVector; }
|
||||
|
||||
public Derivation child(int i) { return children.get(i); }
|
||||
public String childStringValue(int i) {
|
||||
|
|
@ -191,6 +208,11 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
return cat.equals(Rule.rootCat) && ((start == 0 && end == numTokens) || (start == -1));
|
||||
}
|
||||
|
||||
// Return whether |deriv| has root category (for floating parser)
|
||||
public boolean isRootCat() {
|
||||
return cat.equals(Rule.rootCat);
|
||||
}
|
||||
|
||||
// Functions that operate on features.
|
||||
public void addFeature(String domain, String name) { addFeature(domain, name, 1); }
|
||||
public void addFeature(String domain, String name, double value) { this.localFeatureVector.add(domain, name, value); }
|
||||
|
|
@ -234,6 +256,8 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
public void ensureExecuted(Executor executor, ContextValue context) {
|
||||
if (isExecuted()) return;
|
||||
StopWatchSet.begin("Executor.execute");
|
||||
if (opts.showExecutions)
|
||||
LogInfo.logs("%s - %s", canonicalUtterance, formula);
|
||||
Executor.Response response = executor.execute(formula, context);
|
||||
StopWatchSet.end();
|
||||
value = response.value;
|
||||
|
|
@ -304,7 +328,7 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
}
|
||||
|
||||
public String startEndString(List<String> tokens) {
|
||||
return start + ":" + end + tokens.subList(start, end);
|
||||
return start + ":" + end + (start == -1 ? "" : tokens.subList(start, end));
|
||||
}
|
||||
public String toString() { return toLispTree().toString(); }
|
||||
|
||||
|
|
@ -319,8 +343,13 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
for (Derivation child : children)
|
||||
child.incrementAllFeatureVector(factor, map, updateFeatureMatcher);
|
||||
}
|
||||
public void incrementAllFeatureVector(double factor, FeatureVector fv) {
|
||||
localFeatureVector.add(factor, fv);
|
||||
for (Derivation child : children)
|
||||
child.incrementAllFeatureVector(factor, fv);
|
||||
}
|
||||
|
||||
// recursively renames all features in derivation by adding a prefix
|
||||
// returns feature vector with renamed features by prefix
|
||||
public FeatureVector addPrefixLocalFeatureVector(String prefix) {
|
||||
return localFeatureVector.addPrefix(prefix);
|
||||
}
|
||||
|
|
@ -404,4 +433,42 @@ public class Derivation implements SemanticFn.Callable, HasScore {
|
|||
NumUtils.expNormalize(probs);
|
||||
return probs;
|
||||
}
|
||||
|
||||
// Manipulation of temporary state used during parsing.
|
||||
public Map<String, Object> getTempState() {
|
||||
// Create the tempState if it doesn't exist.
|
||||
if (tempState == null)
|
||||
tempState = new HashMap<String, Object>();
|
||||
return tempState;
|
||||
}
|
||||
public void clearTempState() {
|
||||
tempState = null;
|
||||
if (children != null)
|
||||
for (Derivation child : children)
|
||||
child.clearTempState();
|
||||
}
|
||||
|
||||
// Compute anchoredTokens and return the result
|
||||
// anchoredTokens[>= anchoredTokens.length] are False by default
|
||||
public boolean[] getAnchoredTokens() {
|
||||
if (anchoredTokens == null) {
|
||||
if (rule.isAnchored()) {
|
||||
anchoredTokens = new boolean[end];
|
||||
for (int i = start; i < end; i++) anchoredTokens[i] = true;
|
||||
} else {
|
||||
anchoredTokens = new boolean[0];
|
||||
for (Derivation child : children) {
|
||||
boolean[] childAnchoredTokens = child.getAnchoredTokens();
|
||||
if (anchoredTokens.length < childAnchoredTokens.length) {
|
||||
boolean[] newAnchoredTokens = new boolean[childAnchoredTokens.length];
|
||||
for (int i = 0; i < anchoredTokens.length; i++) newAnchoredTokens[i] = anchoredTokens[i];
|
||||
anchoredTokens = newAnchoredTokens;
|
||||
}
|
||||
for (int i = 0; i < childAnchoredTokens.length; i++)
|
||||
anchoredTokens[i] = anchoredTokens[i] || childAnchoredTokens[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
return anchoredTokens.clone();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,214 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import fig.basic.*;
|
||||
|
||||
/**
|
||||
* Prune derivations during parsing.
|
||||
*
|
||||
* To add custom pruning criteria, implement a DerivationPruningComputer class,
|
||||
* and put the class name in the |pruningComputers| option.
|
||||
*
|
||||
* @author ppasupat
|
||||
*/
|
||||
|
||||
public class DerivationPruner {
|
||||
public static class Options {
|
||||
@Option public List<String> pruningStrategies = new ArrayList<>();
|
||||
@Option public List<String> pruningComputers = new ArrayList<>();
|
||||
@Option public int pruningVerbosity = 0;
|
||||
@Option public int maxNumValues = 10;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
public final Parser parser;
|
||||
public final Example ex;
|
||||
private List<DerivationPruningComputer> pruningComputers = new ArrayList<>();
|
||||
private List<String> customAllowedDomains;
|
||||
|
||||
public DerivationPruner(ParserState parserState) {
|
||||
this.parser = parserState.parser;
|
||||
this.ex = parserState.ex;
|
||||
for (String pruningComputer : opts.pruningComputers) {
|
||||
try {
|
||||
Class<?> pruningComputerClass = Class.forName(SempreUtils.resolveClassName(pruningComputer));
|
||||
pruningComputers.add((DerivationPruningComputer) pruningComputerClass.getConstructor(this.getClass()).newInstance(this));
|
||||
} catch (ClassNotFoundException e1) {
|
||||
throw new RuntimeException("Illegal pruning computer: " + pruningComputer);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
e.getCause().printStackTrace();
|
||||
throw new RuntimeException("Error while instantiating pruning computer: " + pruningComputer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void setCustomAllowedDomains(List<String> customAllowedDomains) {
|
||||
this.customAllowedDomains = customAllowedDomains;
|
||||
}
|
||||
|
||||
protected boolean containsStrategy(String name) {
|
||||
return opts.pruningStrategies.contains(name) &&
|
||||
(customAllowedDomains == null || customAllowedDomains.contains(name));
|
||||
}
|
||||
|
||||
public boolean isPruned(Derivation deriv) {
|
||||
if (opts.pruningStrategies.isEmpty() && pruningComputers.isEmpty()) return false;
|
||||
if (pruneFormula(deriv)) return true;
|
||||
if (pruneDenotation(deriv)) return true;
|
||||
for (DerivationPruningComputer pruningComputer : pruningComputers) {
|
||||
if (pruningComputer.isPruned(deriv)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Formula-based Pruning
|
||||
// ============================================================
|
||||
|
||||
private boolean pruneFormula(Derivation deriv) {
|
||||
return pruneSingleton(deriv) || pruneSuperlatives(deriv) || pruneMerges(deriv);
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune singleton formula at the root.
|
||||
*/
|
||||
private boolean pruneSingleton(Derivation deriv) {
|
||||
if (!containsStrategy("singleton")) return false;
|
||||
return deriv.isRoot(ex.numTokens()) && deriv.formula instanceof ValueFormula;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune strings of multiple superlatives.
|
||||
*/
|
||||
private boolean pruneSuperlatives(Derivation deriv) {
|
||||
if (containsStrategy("doubleSuperlatives")) {
|
||||
// Prune if there is an arg{max|min} whose head has arg{max|min}
|
||||
if (deriv.formula instanceof SuperlativeFormula) {
|
||||
SuperlativeFormula superlative = (SuperlativeFormula) deriv.formula;
|
||||
if (superlative.head instanceof SuperlativeFormula) {
|
||||
if (opts.pruningVerbosity >= 2)
|
||||
LogInfo.logs("PRUNED [doubleSuperlatives] %s", deriv.formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (containsStrategy("multipleSuperlatives")) {
|
||||
// Prune if there are more than arg{max|min} appearing in the formula (don't need to be adjacent)
|
||||
List<LispTree> stack = new ArrayList<>();
|
||||
int count = 0;
|
||||
stack.add(deriv.formula.toLispTree());
|
||||
while (!stack.isEmpty()) {
|
||||
LispTree tree = stack.remove(stack.size() - 1);
|
||||
if (tree.isLeaf()) {
|
||||
if ("argmax".equals(tree.value) || "argmin".equals(tree.value)) {
|
||||
count++;
|
||||
if (count >= 2) {
|
||||
if (opts.pruningVerbosity >= 2)
|
||||
LogInfo.logs("PRUNED [multipleSuperlatives] %s", deriv.formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (LispTree subtree : tree.children)
|
||||
stack.add(subtree);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune merges.
|
||||
*/
|
||||
private boolean pruneMerges(Derivation deriv) {
|
||||
if (!(deriv.formula instanceof MergeFormula)) return false;
|
||||
MergeFormula merge = (MergeFormula) deriv.formula;
|
||||
if (containsStrategy("sameMerge")) {
|
||||
if (merge.child1.equals(merge.child2)) {
|
||||
if (opts.pruningVerbosity >= 2)
|
||||
LogInfo.logs("PRUNED [sameMerge] %s", deriv.formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Denotation-based Pruning
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Pruning based on denotations.
|
||||
*/
|
||||
private boolean pruneDenotation(Derivation deriv) {
|
||||
return pruneFinalDenotation(deriv) || prunePartialDenotation(deriv);
|
||||
}
|
||||
|
||||
private boolean pruneFinalDenotation(Derivation deriv) {
|
||||
Formula formula = deriv.formula;
|
||||
// Prune if the denotation is an empty list
|
||||
if (containsStrategy("emptyDenotation")) {
|
||||
deriv.ensureExecuted(parser.executor, ex.context);
|
||||
if (deriv.value instanceof ListValue) {
|
||||
if (((ListValue) deriv.value).values.isEmpty()) {
|
||||
if (opts.pruningVerbosity >= 3)
|
||||
LogInfo.logs("PRUNED [emptyDenotation] %s", formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Prune if the denotation is an error and the formula is not a partial formula
|
||||
if (containsStrategy("nonLambdaError") && !(deriv.formula instanceof LambdaFormula)) {
|
||||
deriv.ensureExecuted(parser.executor, ex.context);
|
||||
if (deriv.value instanceof ErrorValue) {
|
||||
if (opts.pruningVerbosity >= 3)
|
||||
LogInfo.logs("PRUNED [nonLambdaError] %s", formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// Prune if the denotation has too many values
|
||||
if (containsStrategy("tooManyValues") && deriv.isRoot(ex.numTokens())) {
|
||||
deriv.ensureExecuted(parser.executor, ex.context);
|
||||
if (deriv.value instanceof ListValue) {
|
||||
if (((ListValue) deriv.value).values.size() > opts.maxNumValues) {
|
||||
if (opts.pruningVerbosity >= 3)
|
||||
LogInfo.logs("PRUNED [tooManyValues] %s", formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean prunePartialDenotation(Derivation deriv) {
|
||||
Formula formula = deriv.formula;
|
||||
if (containsStrategy("badSuperlativeHead")) {
|
||||
Formula head = null;
|
||||
if (formula instanceof AggregateFormula)
|
||||
head = ((AggregateFormula) formula).child;
|
||||
else if (formula instanceof SuperlativeFormula)
|
||||
head = ((SuperlativeFormula) formula).head;
|
||||
if (head != null) {
|
||||
Value headValue = parser.executor.execute(head, ex.context).value;
|
||||
if (headValue instanceof ListValue && ((ListValue) headValue).values.size() < 2) {
|
||||
if (opts.pruningVerbosity >= 3)
|
||||
LogInfo.logs("PRUNED [badSuperlativeHead] %s", formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (containsStrategy("mistypedMerge") && formula instanceof MergeFormula) {
|
||||
MergeFormula merge = (MergeFormula) formula;
|
||||
SemType type1 = TypeInference.inferType(merge.child1);
|
||||
SemType type2 = TypeInference.inferType(merge.child2);
|
||||
if (!type1.meet(type2).isValid()) {
|
||||
if (opts.pruningVerbosity >= 2)
|
||||
LogInfo.logs("PRUNED [mistypedMerge] %s", deriv.formula);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
/**
|
||||
* Used to prune formulas during parsing.
|
||||
*
|
||||
* Extend this class to add custom pruning criteria,
|
||||
* then add the class name to the |pruningComputers| options of DerivationPruner.
|
||||
*
|
||||
* @author ppasupat
|
||||
*/
|
||||
public abstract class DerivationPruningComputer {
|
||||
|
||||
protected final DerivationPruner pruner;
|
||||
|
||||
public DerivationPruningComputer(DerivationPruner pruner) {
|
||||
this.pruner = pruner;
|
||||
}
|
||||
|
||||
protected boolean containsStrategy(String name) {
|
||||
return pruner.containsStrategy(name);
|
||||
}
|
||||
|
||||
public abstract boolean isPruned(Derivation deriv);
|
||||
|
||||
}
|
||||
|
|
@ -10,9 +10,7 @@ import fig.basic.Evaluation;
|
|||
import fig.basic.LispTree;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* An example corresponds roughly to an input-output pair, the basic unit which
|
||||
|
|
@ -41,13 +39,16 @@ public class Example {
|
|||
@JsonProperty public Value targetValue; // Denotation (e.g., answer)
|
||||
|
||||
//// Information after preprocessing (e.g., tokenization, POS tagging, NER, syntactic parsing, etc.).
|
||||
@JsonProperty public LanguageInfo languageInfo = null;
|
||||
public LanguageInfo languageInfo = null;
|
||||
|
||||
//// Output of the parser.
|
||||
|
||||
// Predicted derivations (sorted by score).
|
||||
public List<Derivation> predDerivations;
|
||||
|
||||
// Temporary state while parsing an Example (see Derivation.java for analogous struture).
|
||||
private Map<String, Object> tempState;
|
||||
|
||||
// Statistics relating to processing the example.
|
||||
public Evaluation evaluation;
|
||||
|
||||
|
|
@ -133,6 +134,8 @@ public class Example {
|
|||
b.setId(arg.child(1).value);
|
||||
} else if ("utterance".equals(label)) {
|
||||
b.setUtterance(arg.child(1).value);
|
||||
} else if ("canonicalUtterance".equals(label)) {
|
||||
b.setUtterance(arg.child(1).value);
|
||||
} else if ("targetFormula".equals(label)) {
|
||||
b.setTargetFormula(Formulas.fromLispTree(arg.child(1)));
|
||||
} else if ("targetValue".equals(label) || "targetValues".equals(label)) {
|
||||
|
|
@ -162,7 +165,7 @@ public class Example {
|
|||
ex.predDerivations = new ArrayList<>();
|
||||
for (int j = 1; j < arg.children.size(); j++)
|
||||
ex.predDerivations.add(derivationFromLispTree(arg.child(j)));
|
||||
} else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues", "context").contains(label)) {
|
||||
} else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues", "context", "original").contains(label)) {
|
||||
throw new RuntimeException("Invalid example argument: " + arg);
|
||||
}
|
||||
}
|
||||
|
|
@ -170,8 +173,9 @@ public class Example {
|
|||
return ex;
|
||||
}
|
||||
|
||||
public void preprocess(LanguageAnalyzer analyzer) {
|
||||
this.languageInfo = analyzer.analyze(this.utterance);
|
||||
public void preprocess() {
|
||||
this.languageInfo = LanguageAnalyzer.getSingleton().analyze(this.utterance);
|
||||
this.targetValue = TargetValuePreprocessor.getSingleton().preprocess(this.targetValue);
|
||||
}
|
||||
|
||||
public void log() {
|
||||
|
|
@ -191,11 +195,6 @@ public class Example {
|
|||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
// To save memory
|
||||
public void clearPredDerivations() {
|
||||
predDerivations.clear();
|
||||
}
|
||||
|
||||
public List<Derivation> getCorrectDerivations() {
|
||||
List<Derivation> res = new ArrayList<>();
|
||||
for (Derivation deriv : predDerivations) {
|
||||
|
|
@ -290,4 +289,14 @@ public class Example {
|
|||
|
||||
return item;
|
||||
}
|
||||
|
||||
public Map<String, Object> getTempState() {
|
||||
// Create the tempState if it doesn't exist.
|
||||
if (tempState == null)
|
||||
tempState = new HashMap<String, Object>();
|
||||
return tempState;
|
||||
}
|
||||
public void clearTempState() {
|
||||
tempState = null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,18 @@ public final class ExampleUtils {
|
|||
out.close();
|
||||
}
|
||||
|
||||
public static void writeJson(List<Example> examples, String outPath) throws IOException {
|
||||
PrintWriter out = edu.stanford.nlp.io.IOUtils.getPrintWriter(outPath);
|
||||
out.println("[");
|
||||
for (int i = 0; i < examples.size(); ++i) {
|
||||
Example ex = examples.get(i);
|
||||
out.print(ex.toJson());
|
||||
out.println(i < examples.size() - 1 ? "," : "");
|
||||
}
|
||||
out.println("]");
|
||||
out.close();
|
||||
}
|
||||
|
||||
private static String escapeSpace(String s) {
|
||||
return s.replaceAll(" ", " ");
|
||||
}
|
||||
|
|
@ -38,6 +50,7 @@ public final class ExampleUtils {
|
|||
PrintWriter out = IOUtils.openOutHard(outPath);
|
||||
|
||||
LispTree p = LispTree.proto;
|
||||
out.println("# SDF version 1.1");
|
||||
out.println("# " + p.L(p.L("iter", iter), p.L("group", group), p.L("numExamples", examples.size()), p.L("evaluation", evaluation.toLispTree())));
|
||||
for (Example ex : examples) {
|
||||
out.println("");
|
||||
|
|
@ -47,11 +60,22 @@ public final class ExampleUtils {
|
|||
if (outputPredDerivations) {
|
||||
for (Derivation deriv : ex.predDerivations) {
|
||||
StringBuilder buf = new StringBuilder();
|
||||
buf.append("item " + escapeSpace(p.L(p.L("formula", deriv.formula.toLispTree()), p.L("value", deriv.value.toLispTree())).toString())); // Description
|
||||
buf.append(" " + deriv.compatibility);
|
||||
buf.append("item");
|
||||
LispTree description = p.newList();
|
||||
if (deriv.canonicalUtterance != null)
|
||||
description.addChild(p.L("canonicalUtterance", deriv.canonicalUtterance));
|
||||
description.addChild(p.L("formula", deriv.formula.toLispTree()));
|
||||
description.addChild(p.L("value", deriv.value.toLispTree()));
|
||||
buf.append("\t" + description);
|
||||
buf.append("\t" + deriv.compatibility);
|
||||
Map<String, Double> features = deriv.getAllFeatureVector();
|
||||
buf.append("\t");
|
||||
boolean first = true;
|
||||
for (Map.Entry<String, Double> e : features.entrySet()) {
|
||||
buf.append(" " + escapeSpace(e.getKey()) + ":" + e.getValue());
|
||||
if (!first)
|
||||
buf.append(' ');
|
||||
first = false;
|
||||
buf.append(e.getKey() + ":" + e.getValue());
|
||||
}
|
||||
out.println(buf.toString());
|
||||
}
|
||||
|
|
@ -60,4 +84,53 @@ public final class ExampleUtils {
|
|||
out.close();
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
public static void writeParaphraseSDF(int iter, String group, Example ex,
|
||||
boolean outputPredDerivations) {
|
||||
String basePath = "preds-iter" + iter + "-" + group + ".examples";
|
||||
String outPath = Execution.getFile(basePath);
|
||||
if (outPath == null) return;
|
||||
PrintWriter out = IOUtils.openOutAppendHard(outPath);
|
||||
|
||||
out.println("example " + ex.id);
|
||||
|
||||
if (outputPredDerivations) {
|
||||
int i = 0;
|
||||
for (Derivation deriv : ex.predDerivations) {
|
||||
if (deriv.canonicalUtterance != null)
|
||||
out.println("Pred@" + i + ":\t" + ex.utterance + "\t" + deriv.canonicalUtterance + "\t" + deriv.compatibility + "\t" + deriv.formula + "\t" + deriv.prob);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
public static void writeEvaluationSDF(int iter, String group,
|
||||
Evaluation evaluation, int numExamples) {
|
||||
String basePath = "preds-iter" + iter + "-" + group + ".examples";
|
||||
String outPath = Execution.getFile(basePath);
|
||||
if (outPath == null) return;
|
||||
PrintWriter out = IOUtils.openOutAppendHard(outPath);
|
||||
|
||||
LispTree p = LispTree.proto;
|
||||
out.println("");
|
||||
out.println("# SDF version 1.1");
|
||||
out.println("# " + p.L(p.L("iter", iter), p.L("group", group), p.L("numExamples", numExamples), p.L("evaluation", evaluation.toLispTree())));
|
||||
out.close();
|
||||
}
|
||||
|
||||
//read lisptree and write json
|
||||
public static void main(String[] args) {
|
||||
Dataset dataset = new Dataset();
|
||||
Pair<String, String> pair = Pair.newPair("train", args[0]);
|
||||
Dataset.opts.splitDevFromTrain = false;
|
||||
dataset.readFromPathPairs(Collections.singletonList(pair));
|
||||
List<Example> examples = dataset.examples("train");
|
||||
try {
|
||||
writeJson(examples, args[1]);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import java.util.*;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.Sets;
|
||||
import fig.basic.*;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* A FeatureExtractor specifies a mapping from derivations to feature vectors.
|
||||
*
|
||||
|
|
@ -34,6 +35,8 @@ public class FeatureExtractor {
|
|||
public boolean useAllFeatures = false;
|
||||
@Option(gloss = "For bigram features in paraphrased utterances, maximum distance to consider")
|
||||
public int maxBigramDistance = 3;
|
||||
@Option(gloss = "Whether or not paraphrasing and bigram features should be lexicalized")
|
||||
public boolean lexicalBigramParaphrase = true;
|
||||
}
|
||||
|
||||
private Executor executor;
|
||||
|
|
@ -63,7 +66,6 @@ public class FeatureExtractor {
|
|||
extractDependencyFeatures(ex, deriv);
|
||||
extractWhTypeFeatures(ex, deriv);
|
||||
conjoinLemmaAndBinary(ex, deriv);
|
||||
extractParaphraseFeatures(ex, deriv);
|
||||
extractBigramFeatures(ex, deriv);
|
||||
for (FeatureComputer featureComputer : featureComputers)
|
||||
featureComputer.extractLocal(ex, deriv);
|
||||
|
|
@ -100,21 +102,24 @@ public class FeatureExtractor {
|
|||
return;
|
||||
}
|
||||
|
||||
if (deriv.value instanceof StringValue) {
|
||||
if (((StringValue) deriv.value).value.equals("[]") || ((StringValue) deriv.value).value.equals("[null]"))
|
||||
deriv.addFeature("denotation", "empty");
|
||||
return;
|
||||
}
|
||||
|
||||
if (deriv.value instanceof ListValue) {
|
||||
ListValue list = (ListValue) deriv.value;
|
||||
|
||||
// TODO(pliang): this is hacky; don't depend on the logical form
|
||||
if (Formulas.isCountFormula(deriv.formula)) {
|
||||
if (list.values.size() != 1) {
|
||||
deriv.addFeature("denotation", "size", list.values.size());
|
||||
} else {
|
||||
int count = getNumber(list.values.get(0));
|
||||
deriv.addFeature("denotation", "count-size" + (count == 0 ? "=0" : ">0"));
|
||||
}
|
||||
} else {
|
||||
if (list.values.size() == 1 && list.values.get(0) instanceof NumberValue) {
|
||||
int count = getNumber(list.values.get(0));
|
||||
deriv.addFeature("denotation", "count-size" + (count <= 1 ? "=" + count : ">1"));
|
||||
}
|
||||
else {
|
||||
int size = list.values.size();
|
||||
deriv.addFeature("denotation", "size" + (size < 3 ? "=" + size : ">=" + 3));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -137,11 +142,11 @@ public class FeatureExtractor {
|
|||
String containment = deriv.containsIndex(dependency.modifier) ? "internal" : "external";
|
||||
if (containsDomain("fullDependencyParse"))
|
||||
addAllDependencyFeatures(dependency, direction, containment,
|
||||
deriv);
|
||||
deriv);
|
||||
else
|
||||
deriv.addFeature("dependencyParse",
|
||||
"(" + dependency.label + " " + direction + " " + containment + ") --- "
|
||||
+ deriv.getRule().toString());
|
||||
"(" + dependency.label + " " + direction + " " + containment + ") --- "
|
||||
+ deriv.getRule().toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -150,7 +155,7 @@ public class FeatureExtractor {
|
|||
}
|
||||
|
||||
private void addAllDependencyFeatures(LanguageInfo.DependencyEdge dependency,
|
||||
String direction, String containment, Derivation deriv) {
|
||||
String direction, String containment, Derivation deriv) {
|
||||
String[] types = {dependency.label, "*"};
|
||||
String[] directions = {" " + direction, ""};
|
||||
String[] containments = {" " + containment, ""};
|
||||
|
|
@ -160,7 +165,7 @@ public class FeatureExtractor {
|
|||
for (String containmentPresent : containments) {
|
||||
for (String rulePresent : rules) {
|
||||
deriv.addFeature("fullDependencyParse",
|
||||
"(" + typePresent + directionPresent + containmentPresent + ") --- " + rulePresent);
|
||||
"(" + typePresent + directionPresent + containmentPresent + ") --- " + rulePresent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -175,8 +180,8 @@ public class FeatureExtractor {
|
|||
|
||||
if (ex.posTag(0).startsWith("W")) {
|
||||
deriv.addFeature("whType",
|
||||
"token0=" + ex.token(0) + "," +
|
||||
"type=" + coarseType(deriv.type.toString()));
|
||||
"token0=" + ex.token(0) + "," +
|
||||
"type=" + coarseType(deriv.type.toString()));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -229,7 +234,7 @@ public class FeatureExtractor {
|
|||
//Used in Berant et., EMNLP 2013, and in the agenda RL parser
|
||||
//Extracts all content-word lemmas in the derivation tree not dominated by the category $Entity
|
||||
private void extractNonEntityLemmas(Example ex, Derivation deriv,
|
||||
List<String> nonEntityLemmas) {
|
||||
List<String> nonEntityLemmas) {
|
||||
if (deriv.children.size() == 0) { // base case this means it is a word that should be appended
|
||||
for (int i = deriv.start; i < deriv.end; i++) {
|
||||
String pos = ex.languageInfo.posTags.get(i);
|
||||
|
|
@ -259,31 +264,6 @@ public class FeatureExtractor {
|
|||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an indicator for each token alignment between a token in the utterance and its
|
||||
* canonical form in the grammar using the ParaphraseModel, if applicable.
|
||||
*/
|
||||
// Compute paraphrasing features. In the future, need to store temporary
|
||||
// state to make this more efficient.
|
||||
void extractParaphraseFeatures(Example ex, Derivation deriv) {
|
||||
if (!containsDomain("paraphrasing")) return;
|
||||
if (deriv.rule == Rule.nullRule) return;
|
||||
extractParaphrasePrecision(ex, deriv);
|
||||
extractParaphraseRecall(ex, deriv);
|
||||
|
||||
// Make sure we have a valid, floating, paraphraseable rule
|
||||
if (!deriv.rule.isFloating() || !deriv.rule.isRhsTerminals()) return;
|
||||
String key = join(deriv.rule.rhs, " ");
|
||||
ParaphraseModel model = ParaphraseModel.getSingleton();
|
||||
if (!model.containsKey(key)) return;
|
||||
for (String token : ex.getTokens()) {
|
||||
double score = model.get(key, token);
|
||||
if (score > 0.0) {
|
||||
deriv.addFeature("paraphrasing", "(" + key + ", " + token + ")");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an indicator for each pair of bigrams that can be aligned from the original
|
||||
* utterance and two (not necessarily contiguous) lemmas in the generated utterance
|
||||
|
|
@ -294,45 +274,26 @@ public class FeatureExtractor {
|
|||
LanguageInfo derivInfo = LanguageAnalyzer.getSingleton().analyze(deriv.canonicalUtterance);
|
||||
List<String> derivLemmas = derivInfo.lemmaTokens;
|
||||
List<String> exLemmas = ex.languageInfo.lemmaTokens;
|
||||
Map<Integer, Integer> bigramCounts = new HashMap<Integer, Integer>();
|
||||
for (int i = 0; i < exLemmas.size() - 1; i++) {
|
||||
for (int j = 0; j < derivLemmas.size() - 1; j++) {
|
||||
if (derivLemmas.get(j).equals(exLemmas.get(i))) {
|
||||
// Consider bigrams separated by up to maxBigramDistance in generated utterance
|
||||
for (int k = 1; j + k < derivLemmas.size() && k <= opts.maxBigramDistance; k++) {
|
||||
if (derivLemmas.get(j + k).equals(exLemmas.get(i + 1))) {
|
||||
deriv.addFeature("bigram",
|
||||
exLemmas.get(i) + "," + exLemmas.get(i + 1) + " - " + k);
|
||||
if (opts.lexicalBigramParaphrase)
|
||||
deriv.addFeature("bigram",
|
||||
exLemmas.get(i) + "," + exLemmas.get(i + 1) + " - " + k);
|
||||
else MapUtils.incr(bigramCounts, k, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void extractParaphraseRecall(Example ex, Derivation deriv) {
|
||||
if (!deriv.cat.equals(Rule.rootCat)) return;
|
||||
String[] derivTokens = deriv.canonicalUtterance.split("\\s+");
|
||||
int n = 0;
|
||||
for (int i = 0; i < ex.languageInfo.numTokens(); ++i) {
|
||||
for (int j = 0; j < derivTokens.length; ++j) {
|
||||
if (derivTokens[j].equals(ex.languageInfo.tokens.get(i))) {
|
||||
n++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!opts.lexicalBigramParaphrase) {
|
||||
for (Integer dist : bigramCounts.keySet())
|
||||
deriv.addFeature("bigram", "distance " + dist + " - " + bigramCounts.get(dist));
|
||||
}
|
||||
deriv.addFeature("paraphrasing", "deriv_recall", (double) n / ex.languageInfo.numTokens());
|
||||
}
|
||||
|
||||
private void extractParaphrasePrecision(Example ex, Derivation deriv) {
|
||||
//how many of the generated stuff is in the original utterance
|
||||
int n = 0;
|
||||
for (String item : deriv.rule.rhs) {
|
||||
if (Rule.isCat(item)) continue;
|
||||
if (ex.languageInfo.tokens.contains(item))
|
||||
n++;
|
||||
}
|
||||
deriv.addFeature("paraphrasing", "deriv_prec", n);
|
||||
}
|
||||
|
||||
// Joins arrayList of strings into string
|
||||
|
|
|
|||
|
|
@ -17,6 +17,14 @@ import java.util.*;
|
|||
* @author Jonathan Berant
|
||||
*/
|
||||
public class FeatureVector {
|
||||
public static class Options {
|
||||
@Option(gloss = "When logging, ignore features with zero weight")
|
||||
public boolean ignoreZeroWeight = false;
|
||||
@Option(gloss = "Log only this number of top and bottom features")
|
||||
public int logFeaturesLimit = Integer.MAX_VALUE;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
// These features map to the value 1 (most common case in NLP).
|
||||
private ArrayList<String> indicatorFeatures;
|
||||
// General features
|
||||
|
|
@ -90,21 +98,27 @@ public class FeatureVector {
|
|||
}
|
||||
|
||||
public void add(FeatureVector that) { add(that, AllFeatureMatcher.matcher); }
|
||||
public void add(FeatureVector that, FeatureMatcher matcher) {
|
||||
public void add(double scale, FeatureVector that) { add(scale, that, AllFeatureMatcher.matcher); }
|
||||
public void add(FeatureVector that, FeatureMatcher matcher) { add(1, that, matcher); }
|
||||
public void add(double scale, FeatureVector that, FeatureMatcher matcher) {
|
||||
if (that.indicatorFeatures != null) {
|
||||
for (String f : that.indicatorFeatures)
|
||||
if (matcher.matches(f))
|
||||
add(f);
|
||||
if (matcher.matches(f)) {
|
||||
if (scale == 1)
|
||||
add(f);
|
||||
else
|
||||
add(f, scale);
|
||||
}
|
||||
}
|
||||
if (that.generalFeatures != null) {
|
||||
for (Pair<String, Double> pair : that.generalFeatures)
|
||||
if (matcher.matches(pair.getFirst()))
|
||||
add(pair.getFirst(), pair.getSecond());
|
||||
add(pair.getFirst(), scale * pair.getSecond());
|
||||
}
|
||||
// dense features are always added
|
||||
if (that.denseFeatures != null) {
|
||||
for (int i = 0; i < denseFeatures.length; ++i)
|
||||
denseFeatures[i] = that.denseFeatures[i];
|
||||
denseFeatures[i] += scale * that.denseFeatures[i];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -229,16 +243,33 @@ public class FeatureVector {
|
|||
String feature = entry.getKey();
|
||||
if (entry.getValue() == 0) continue;
|
||||
double value = entry.getValue() * params.getWeight(feature);
|
||||
if (opts.ignoreZeroWeight && value == 0) continue;
|
||||
sumValue += value;
|
||||
entries.add(new java.util.AbstractMap.SimpleEntry<String, Double>(feature, value));
|
||||
}
|
||||
Collections.sort(entries, new ValueComparator<String, Double>(false));
|
||||
LogInfo.begin_track_printAll("%s features [sum = %s] (format is feature value * weight)", prefix, Fmt.D(sumValue));
|
||||
for (Map.Entry<String, Double> entry : entries) {
|
||||
String feature = entry.getKey();
|
||||
double value = entry.getValue();
|
||||
double weight = params.getWeight(feature);
|
||||
LogInfo.logs("%-50s %6s = %s * %s", "[ " + feature + " ]", Fmt.D(value), Fmt.D(MapUtils.getDouble(features, feature, 0)), Fmt.D(weight));
|
||||
if (entries.size() / 2 > opts.logFeaturesLimit) {
|
||||
for (Map.Entry<String, Double> entry : entries.subList(0, opts.logFeaturesLimit)) {
|
||||
String feature = entry.getKey();
|
||||
double value = entry.getValue();
|
||||
double weight = params.getWeight(feature);
|
||||
LogInfo.logs("%-50s %6s = %s * %s", "[ " + feature + " ]", Fmt.D(value), Fmt.D(MapUtils.getDouble(features, feature, 0)), Fmt.D(weight));
|
||||
}
|
||||
LogInfo.logs("... (%d more features) ...", entries.size() - 2 * opts.logFeaturesLimit);
|
||||
for (Map.Entry<String, Double> entry : entries.subList(entries.size() - opts.logFeaturesLimit, entries.size())) {
|
||||
String feature = entry.getKey();
|
||||
double value = entry.getValue();
|
||||
double weight = params.getWeight(feature);
|
||||
LogInfo.logs("%-50s %6s = %s * %s", "[ " + feature + " ]", Fmt.D(value), Fmt.D(MapUtils.getDouble(features, feature, 0)), Fmt.D(weight));
|
||||
}
|
||||
} else {
|
||||
for (Map.Entry<String, Double> entry : entries) {
|
||||
String feature = entry.getKey();
|
||||
double value = entry.getValue();
|
||||
double weight = params.getWeight(feature);
|
||||
LogInfo.logs("%-50s %6s = %s * %s", "[ " + feature + " ]", Fmt.D(value), Fmt.D(MapUtils.getDouble(features, feature, 0)), Fmt.D(weight));
|
||||
}
|
||||
}
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import fig.basic.LispTree;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Given a token at a particular position, keep it is from a select set.
|
||||
*
|
||||
* @author ppasupat
|
||||
*/
|
||||
public class FilterTokenFn extends SemanticFn {
|
||||
List<String> acceptableTokens = new ArrayList<>();
|
||||
String mode;
|
||||
|
||||
public void init(LispTree tree) {
|
||||
super.init(tree);
|
||||
mode = tree.child(1).value;
|
||||
if (!mode.equals("token") && !mode.equals("lemma"))
|
||||
throw new RuntimeException("Illegal description for FilterTokenFn: " + mode);
|
||||
for (int j = 2; j < tree.children.size(); j++) {
|
||||
acceptableTokens.add(tree.child(j).value);
|
||||
}
|
||||
}
|
||||
|
||||
public DerivationStream call(final Example ex, final Callable c) {
|
||||
return new SingleDerivationStream() {
|
||||
@Override
|
||||
public Derivation createDerivation() {
|
||||
if (!isValid(ex, c))
|
||||
return null;
|
||||
else {
|
||||
return new Derivation.Builder()
|
||||
.withCallable(c)
|
||||
.withFormulaFrom(c.child(0))
|
||||
.createDerivation();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private boolean isValid(Example ex, Callable c) {
|
||||
if (c.getEnd() - c.getStart() != 1) return false;
|
||||
String token;
|
||||
if ("token".equals(mode))
|
||||
token = ex.token(c.getStart());
|
||||
else
|
||||
token = ex.lemmaToken(c.getStart());
|
||||
return acceptableTokens.contains(token);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import fig.basic.*;
|
||||
import fig.exec.Execution;
|
||||
|
||||
import java.io.PrintWriter;
|
||||
import java.util.*;
|
||||
|
||||
import static fig.basic.LogInfo.logs;
|
||||
|
|
@ -46,7 +48,18 @@ public class FloatingParser extends Parser {
|
|||
@Option public boolean defaultIsFloating = true;
|
||||
@Option (gloss = "Flag specifying whether anchored spans/tokens can only be used once in a derivation")
|
||||
public boolean useAnchorsOnce = false;
|
||||
@Option (gloss = "Flag specifying whether floating rules are allowed to be applied consecutively")
|
||||
public boolean consecutiveRules = true;
|
||||
@Option (gloss = "Whether to always execute the derivation")
|
||||
public boolean executeAllDerivations = false;
|
||||
@Option (gloss = "Whether to output a file with all utterances predicted")
|
||||
public boolean printPredictedUtterances = false;
|
||||
@Option(gloss = "Put a limit on formula size instead of formula depth")
|
||||
public boolean useSizeInsteadOfDepth = false;
|
||||
@Option(gloss = "Custom beam size at training time (default = Parser.beamSize)")
|
||||
public int trainBeamSize = -1;
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
||||
public FloatingParser(Spec spec) { super(spec); }
|
||||
|
|
@ -70,12 +83,25 @@ class FloatingParserState extends ParserState {
|
|||
// Examples of state:
|
||||
// (category, depth)
|
||||
// (category, depth, set of tokens)
|
||||
|
||||
private final Map<Object, List<Derivation>> chart = new HashMap<>();
|
||||
|
||||
private final DerivationPruner pruner;
|
||||
|
||||
public FloatingParserState(FloatingParser parser, Params params, Example ex, boolean computeExpectedCounts) {
|
||||
super(parser, params, ex, computeExpectedCounts);
|
||||
pruner = new DerivationPruner(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int getBeamSize() {
|
||||
if (computeExpectedCounts && FloatingParser.opts.trainBeamSize > 0)
|
||||
return FloatingParser.opts.trainBeamSize;
|
||||
return Parser.opts.beamSize;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Construct state names.
|
||||
private Object floatingCell(String cat, int depth) {
|
||||
return cat + ":" + depth;
|
||||
|
|
@ -88,10 +114,11 @@ class FloatingParserState extends ParserState {
|
|||
}
|
||||
|
||||
private void addToChart(Object cell, Derivation deriv) {
|
||||
if (Parser.opts.verbose >= 3)
|
||||
LogInfo.logs("addToChart %s: %s", cell, deriv);
|
||||
if (!deriv.isFeaturizedAndScored()) // A derivation could be belong in multiple cells.
|
||||
featurizeAndScoreDerivation(deriv);
|
||||
if (Parser.opts.pruneErrorValues && deriv.value instanceof ErrorValue) return;
|
||||
if (Parser.opts.verbose >= 4)
|
||||
LogInfo.logs("addToChart %s: %s", cell, deriv);
|
||||
MapUtils.addToList(chart, cell, deriv);
|
||||
}
|
||||
|
||||
|
|
@ -103,29 +130,32 @@ class FloatingParserState extends ParserState {
|
|||
else if (child2 == null) // 1-ary
|
||||
children = Collections.singletonList(child1);
|
||||
else {
|
||||
children = ListUtils.newList(child1, child2);
|
||||
// optionally: ensure that specific anchors are only used once per final derivation
|
||||
// Optional: ensure that each anchor is only used once per derivation.
|
||||
if (FloatingParser.opts.useAnchorsOnce &&
|
||||
FloatingRuleUtils.derivationAnchorsOverlap(child1, child2))
|
||||
FloatingRuleUtils.derivationAnchorsOverlap(child1, child2))
|
||||
return;
|
||||
children = ListUtils.newList(child1, child2);
|
||||
}
|
||||
|
||||
// optionally: ensure that rule being applied is not the same as one of the children's
|
||||
if (!FloatingParser.opts.consecutiveRules) {
|
||||
for (Derivation child : children) {
|
||||
if (child.rule.equals(rule)) return;
|
||||
}
|
||||
}
|
||||
|
||||
DerivationStream results = rule.sem.call(ex,
|
||||
new SemanticFn.CallInfo(rule.lhs, start, end, rule, children));
|
||||
new SemanticFn.CallInfo(rule.lhs, start, end, rule, children));
|
||||
while (results.hasNext()) {
|
||||
Derivation newDeriv = results.next();
|
||||
newDeriv.canonicalUtterance = canonicalUtterance;
|
||||
|
||||
// make sure we execute
|
||||
if (FloatingParser.opts.executeAllDerivations && !(newDeriv.type instanceof FuncSemType))
|
||||
newDeriv.ensureExecuted(parser.executor, ex.context);
|
||||
|
||||
if (pruner.isPruned(newDeriv)) continue;
|
||||
// Avoid repetitive floating cells
|
||||
/*
|
||||
if (depth > 0) {
|
||||
if (MapUtils.getSet(setChart, rule.lhs).contains(newDeriv.formula)) {
|
||||
continue;
|
||||
}
|
||||
if (rule.lhs == Rule.rootCat)
|
||||
LogInfo.logs("adding %s to root for floating", newDeriv);
|
||||
MapUtils.addToSet(setChart, rule.lhs, newDeriv.formula);
|
||||
}
|
||||
*/
|
||||
addToChart(cell(rule.lhs, start, end, depth), newDeriv);
|
||||
if (depth == -1) // In addition, anchored cells become floating at level 0
|
||||
addToChart(floatingCell(rule.lhs, 0), newDeriv);
|
||||
|
|
@ -236,19 +266,30 @@ class FloatingParserState extends ParserState {
|
|||
for (Derivation deriv : derivations)
|
||||
applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2);
|
||||
} else { // $Cat $Cat
|
||||
for (int subDepth = 0; subDepth < depth; subDepth++) { // depth-1 <=depth-1
|
||||
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, depth - 1));
|
||||
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, subDepth));
|
||||
for (Derivation deriv1 : derivations1)
|
||||
for (Derivation deriv2 : derivations2)
|
||||
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
|
||||
}
|
||||
for (int subDepth = 0; subDepth < depth - 1; subDepth++) { // <depth-1 depth-1
|
||||
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, subDepth));
|
||||
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, depth - 1));
|
||||
for (Derivation deriv1 : derivations1)
|
||||
for (Derivation deriv2 : derivations2)
|
||||
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
|
||||
if (FloatingParser.opts.useSizeInsteadOfDepth) {
|
||||
for (int depth1 = 0; depth1 < depth; depth1++) {
|
||||
int depth2 = depth - 1 - depth1;
|
||||
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, depth1));
|
||||
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, depth2));
|
||||
for (Derivation deriv1 : derivations1)
|
||||
for (Derivation deriv2 : derivations2)
|
||||
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
|
||||
}
|
||||
} else {
|
||||
for (int subDepth = 0; subDepth < depth; subDepth++) { // depth-1 <=depth-1
|
||||
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, depth - 1));
|
||||
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, subDepth));
|
||||
for (Derivation deriv1 : derivations1)
|
||||
for (Derivation deriv2 : derivations2)
|
||||
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
|
||||
}
|
||||
for (int subDepth = 0; subDepth < depth - 1; subDepth++) { // <depth-1 depth-1
|
||||
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, subDepth));
|
||||
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, depth - 1));
|
||||
for (Derivation deriv1 : derivations1)
|
||||
for (Derivation deriv2 : derivations2)
|
||||
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -289,14 +330,6 @@ class FloatingParserState extends ParserState {
|
|||
buildAnchored(i, i + len);
|
||||
for (String cat : categories) {
|
||||
String cell = anchoredCell(cat, i, i + len).toString();
|
||||
/*if (cat.equals("$Word") || cat.equals("$Fragment")) {
|
||||
LogInfo.logs("Looking at cell: %s", cell);
|
||||
if (chart.get(cell) == null) LogInfo.logs("???");
|
||||
else {
|
||||
for (Derivation deriv : chart.get(cell))
|
||||
LogInfo.logs("Found cell of interest: %s", deriv.toString());
|
||||
}
|
||||
}*/
|
||||
pruneCell(cell, chart.get(cell));
|
||||
}
|
||||
}
|
||||
|
|
@ -307,14 +340,6 @@ class FloatingParserState extends ParserState {
|
|||
buildFloating(depth);
|
||||
for (String cat : categories) {
|
||||
String cell = floatingCell(cat, depth).toString();
|
||||
/*if (cat.equals("$Word") || cat.equals("$Fragment")) {
|
||||
LogInfo.logs("Looking at floating cell: %s", cell);
|
||||
if (chart.get(cell) == null) LogInfo.logs("???");
|
||||
else {
|
||||
for (Derivation deriv : chart.get(cell))
|
||||
LogInfo.logs("Found cell of interest: %s", deriv.toString());
|
||||
}
|
||||
}*/
|
||||
pruneCell(cell, chart.get(cell));
|
||||
}
|
||||
}
|
||||
|
|
@ -339,9 +364,29 @@ class FloatingParserState extends ParserState {
|
|||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
if (FloatingParser.opts.printPredictedUtterances) {
|
||||
PrintWriter writer = IOUtils.openOutAppendEasy(Execution.getFile("canonical_utterances"));
|
||||
PrintWriter fWriter = IOUtils.openOutAppendEasy(Execution.getFile("utterances_formula.tsv"));
|
||||
Derivation.sortByScore(predDerivations);
|
||||
for (Derivation deriv: predDerivations) {
|
||||
if (deriv.score > -10) {
|
||||
writer.println(String.format("%s\t%s", deriv.canonicalUtterance, deriv.score));
|
||||
fWriter.println(String.format("%s\t%s", deriv.canonicalUtterance, deriv.formula.toString()));
|
||||
}
|
||||
}
|
||||
writer.close();
|
||||
fWriter.close();
|
||||
}
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setEvaluation() {
|
||||
super.setEvaluation();
|
||||
evaluation.add("numCells", chart.size());
|
||||
}
|
||||
|
||||
private void visualizeAnchoredChart(Set<String> categories) {
|
||||
for (String cat : categories) {
|
||||
for (int len = 1; len <= numTokens; ++len) {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ public final class FloatingRuleUtils {
|
|||
* that spans) [2, 4] then we have an overlap.
|
||||
*/
|
||||
public static boolean derivationAnchorsOverlap(Derivation a, Derivation b) {
|
||||
/*
|
||||
List<Derivation> aRoots = getDerivationAnchors(a);
|
||||
List<Derivation> bRoots = getDerivationAnchors(b);
|
||||
for (Derivation aRoot : aRoots) {
|
||||
|
|
@ -40,5 +41,10 @@ public final class FloatingRuleUtils {
|
|||
}
|
||||
}
|
||||
return false;
|
||||
*/
|
||||
boolean[] aAnchors = a.getAnchoredTokens(), bAnchors = b.getAnchoredTokens();
|
||||
for (int i = 0; i < aAnchors.length && i < bAnchors.length; i++)
|
||||
if (aAnchors[i] && bAnchors[i]) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ public abstract class Formula {
|
|||
|
||||
public static Formula nullFormula = new PrimitiveFormula() {
|
||||
public LispTree toLispTree() { return LispTree.proto.newLeaf("null"); }
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override public boolean equals(Object o) { return this == o; }
|
||||
public int computeHashCode() { return 0; }
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import java.util.*;
|
|||
import fig.basic.*;
|
||||
|
||||
/**
|
||||
* Similar to LexiconFn, but list all approximate matches from a TableKnowledgeGraph.
|
||||
* Similar to LexiconFn, but list all approximate matches from a TableKnowledgeGraph or PuzzleKnowledgeGraph.
|
||||
*
|
||||
* @author ppasupat
|
||||
*/
|
||||
|
|
@ -15,7 +15,8 @@ public class FuzzyMatchFn extends SemanticFn {
|
|||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
public enum FuzzyMatchFnMode { UNARY, BINARY, ENTITY };
|
||||
public enum FuzzyMatchFnMode { UNARY, BINARY, ENTITY,
|
||||
ORDER_BEFORE, ORDER_AFTER, ORDER_NEXT, ORDER_PREV, ORDER_ADJACENT };
|
||||
private FuzzyMatchFnMode mode;
|
||||
|
||||
// Generate all possible denotations regardless of the phrase
|
||||
|
|
@ -29,6 +30,11 @@ public class FuzzyMatchFn extends SemanticFn {
|
|||
else if ("binary".equals(value)) this.mode = FuzzyMatchFnMode.BINARY;
|
||||
else if ("entity".equals(value)) this.mode = FuzzyMatchFnMode.ENTITY;
|
||||
else if ("any".equals(value)) this.matchAny = true;
|
||||
else if ("before".equals(value)) this.mode = FuzzyMatchFnMode.ORDER_BEFORE;
|
||||
else if ("after".equals(value)) this.mode = FuzzyMatchFnMode.ORDER_AFTER;
|
||||
else if ("next".equals(value)) this.mode = FuzzyMatchFnMode.ORDER_NEXT;
|
||||
else if ("prev".equals(value)) this.mode = FuzzyMatchFnMode.ORDER_PREV;
|
||||
else if ("adjacent".equals(value)) this.mode = FuzzyMatchFnMode.ORDER_ADJACENT;
|
||||
else throw new RuntimeException("Invalid argument: " + value);
|
||||
}
|
||||
}
|
||||
|
|
@ -57,7 +63,7 @@ public class FuzzyMatchFn extends SemanticFn {
|
|||
this.ex = ex;
|
||||
this.graph = (ex.context == null) ? null : ex.context.graph;
|
||||
this.c = c;
|
||||
this.query = matchAny ? null : c.childStringValue(0);
|
||||
this.query = (matchAny || c.getChildren().isEmpty()) ? null : c.childStringValue(0);
|
||||
this.mode = mode;
|
||||
this.matchAny = matchAny;
|
||||
if (opts.verbose >= 2)
|
||||
|
|
@ -68,6 +74,7 @@ public class FuzzyMatchFn extends SemanticFn {
|
|||
@Override
|
||||
public Derivation createDerivation() {
|
||||
if (graph == null) return null;
|
||||
if (query == null && !matchAny) return null;
|
||||
|
||||
// Compute the formulas if not computed yet
|
||||
if (formulas == null) {
|
||||
|
|
|
|||
|
|
@ -184,6 +184,8 @@ public class Grammar {
|
|||
return !interpretBoolean(tree.child(1), tags);
|
||||
if ("and".equals(tree.child(0).value))
|
||||
return interpretBoolean(tree.child(1), tags) && interpretBoolean(tree.child(2), tags);
|
||||
if ("or".equals(tree.child(0).value))
|
||||
return interpretBoolean(tree.child(1), tags) || interpretBoolean(tree.child(2), tags);
|
||||
throw new RuntimeException("Expected a single tag, but got: " + tree);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@ package edu.stanford.nlp.sempre;
|
|||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import fig.basic.MapUtils;
|
||||
import fig.basic.Option;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.Arrays;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -19,6 +20,12 @@ import java.util.Map;
|
|||
* @author Percy Liang
|
||||
*/
|
||||
public class JavaExecutor extends Executor {
|
||||
public static class Options {
|
||||
@Option(gloss = "Whether to convert NumberValue to int/double") public boolean convertNumberValues = true;
|
||||
@Option(gloss = "Print stack trace on exception") public boolean printStackTrace = false;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
private static JavaExecutor defaultExecutor = new JavaExecutor();
|
||||
|
||||
// To simplify logical forms, define some shortcuts.
|
||||
|
|
@ -78,6 +85,12 @@ public class JavaExecutor extends Executor {
|
|||
public static String plus(String a, String b, String c, String d, String e) {
|
||||
return a + b + c + d + e;
|
||||
}
|
||||
public static String plus(String a, String b, String c, String d, String e, String f) {
|
||||
return a + b + c + d + e + f;
|
||||
}
|
||||
public static String plus(String a, String b, String c, String d, String e, String f, String g) {
|
||||
return a + b + c + d + e + f + g;
|
||||
}
|
||||
private static String toString(Object x) {
|
||||
if (x instanceof String)
|
||||
return (String) x;
|
||||
|
|
@ -146,7 +159,8 @@ public class JavaExecutor extends Executor {
|
|||
try {
|
||||
return new Response(toValue(processFormula(formula)));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
// Comment this out if we expect lots of innocuous type checking failures
|
||||
if (opts.printStackTrace) e.printStackTrace();
|
||||
return new Response(ErrorValue.badJava(e.toString()));
|
||||
}
|
||||
}
|
||||
|
|
@ -156,6 +170,7 @@ public class JavaExecutor extends Executor {
|
|||
return toObject(((ValueFormula) formula).value);
|
||||
|
||||
if (formula instanceof CallFormula) { // Invoke the function.
|
||||
//LogInfo.logs("formula=%s", formula);
|
||||
// Recurse
|
||||
CallFormula call = (CallFormula) formula;
|
||||
Object func = processFormula(call.func);
|
||||
|
|
@ -170,8 +185,9 @@ public class JavaExecutor extends Executor {
|
|||
String id = ((NameValue) func).id;
|
||||
id = MapUtils.get(shortcuts, id, id);
|
||||
|
||||
if (id.startsWith(".")) // Instance method
|
||||
if (id.startsWith(".")) // Instance method
|
||||
return invoke(id.substring(1), args.get(0), args.subList(1, args.size()).toArray(new Object[0]));
|
||||
|
||||
else // Static method
|
||||
return invoke(id, null, args.toArray(new Object[0]));
|
||||
}
|
||||
|
|
@ -199,7 +215,7 @@ public class JavaExecutor extends Executor {
|
|||
// Convert a Value (which are specified in the formulas) to an Object (which
|
||||
// many Java functions take).
|
||||
private static Object toObject(Value value) {
|
||||
if (value instanceof NumberValue) {
|
||||
if (value instanceof NumberValue && opts.convertNumberValues) {
|
||||
// Unfortunately, NumberValues don't make a distinction between ints and
|
||||
// doubles, so this is a hack.
|
||||
double x = ((NumberValue) value).value;
|
||||
|
|
@ -252,6 +268,7 @@ public class JavaExecutor extends Executor {
|
|||
int bestCost = INVALID_TYPE_COST;
|
||||
for (Method m : methods) {
|
||||
if (!m.getName().equals(methodName)) continue;
|
||||
m.setAccessible(true);
|
||||
nameMatches.add(m);
|
||||
if (isStatic != Modifier.isStatic(m.getModifiers())) continue;
|
||||
int cost = typeCastCost(m.getParameterTypes(), args);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ public class JoinFn extends SemanticFn {
|
|||
@Option(gloss = "Verbose") public int verbose = 0;
|
||||
@Option public boolean showTypeCheckFailures = false;
|
||||
@Option public boolean typeInference = false;
|
||||
// TODO(joberant): this flag is for backward compatibility. If we don't
|
||||
// need it for the new results, get rid of it.
|
||||
@Option public boolean specializedTypeCheck = true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ public class JoinFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof JoinFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -25,8 +25,13 @@ public abstract class KnowledgeGraph {
|
|||
String className = tree.child(1).value;
|
||||
Class<?> classObject = Class.forName(SempreUtils.resolveClassName(className));
|
||||
return (KnowledgeGraph) classObject.getDeclaredMethod("fromLispTree", LispTree.class).invoke(null, tree);
|
||||
} catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException
|
||||
| NoSuchMethodException | SecurityException | ClassNotFoundException e) {
|
||||
} catch (InvocationTargetException e) {
|
||||
e.getCause().printStackTrace();
|
||||
LogInfo.fail(e.getCause());
|
||||
throw new RuntimeException(e);
|
||||
} catch (IllegalAccessException | IllegalArgumentException |
|
||||
NoSuchMethodException | SecurityException | ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ public class LambdaFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof LambdaFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ public class LanguageInfo implements MemUsage.Instrumented {
|
|||
public final List<String> nerValues; // NER values (contains times, dates, etc.)
|
||||
|
||||
private Map<String, IntPair> lemmaSpans;
|
||||
private Set<String> lowercasedSpans;
|
||||
|
||||
|
||||
|
||||
public static class DependencyEdge {
|
||||
@JsonProperty
|
||||
|
|
@ -291,7 +294,7 @@ public class LanguageInfo implements MemUsage.Instrumented {
|
|||
|
||||
public Map<String, IntPair> getLemmaSpans() {
|
||||
if (lemmaSpans == null) {
|
||||
lemmaSpans = new HashMap<String, IntPair>();
|
||||
lemmaSpans = new HashMap<>();
|
||||
for (int i = 0; i < numTokens() - 1; ++i) {
|
||||
for (int j = i + 1; j < numTokens(); ++j)
|
||||
lemmaSpans.put(lemmaPhrase(i, j), new IntPair(i, j));
|
||||
|
|
@ -300,6 +303,17 @@ public class LanguageInfo implements MemUsage.Instrumented {
|
|||
return lemmaSpans;
|
||||
}
|
||||
|
||||
public Set<String> getLowerCasedSpans() {
|
||||
if (lowercasedSpans == null) {
|
||||
lowercasedSpans = new HashSet<>();
|
||||
for (int i = 0; i < numTokens() - 1; ++i) {
|
||||
for (int j = i + 1; j < numTokens(); ++j)
|
||||
lowercasedSpans.add(phrase(i, j).toLowerCase());
|
||||
}
|
||||
}
|
||||
return lowercasedSpans;
|
||||
}
|
||||
|
||||
public boolean matchLemmas(List<WordInfo> wordInfos) {
|
||||
for (int i = 0; i < numTokens(); ++i) {
|
||||
if (matchLemmasFromIndex(wordInfos, i))
|
||||
|
|
@ -324,6 +338,7 @@ public class LanguageInfo implements MemUsage.Instrumented {
|
|||
*
|
||||
*/
|
||||
public static class LanguageUtils {
|
||||
|
||||
public static boolean sameProperNounClass(String noun1, String noun2) {
|
||||
if ((noun1.equals("NNP") || noun1.equals("NNPS")) &&
|
||||
(noun2.equals("NNP") || noun2.equals("NNPS")))
|
||||
|
|
@ -335,6 +350,10 @@ public class LanguageInfo implements MemUsage.Instrumented {
|
|||
return pos.startsWith("NNP");
|
||||
}
|
||||
|
||||
public static boolean isSuperlative(String pos) { return pos.equals("RBS") || pos.equals("JJS"); }
|
||||
public static boolean isComparative(String pos) { return pos.equals("RBR") || pos.equals("JJR"); }
|
||||
|
||||
|
||||
public static boolean isEntity(LanguageInfo info, int i) {
|
||||
return isProperNoun(info.posTags.get(i)) || !(info.nerTags.get(i).equals("O"));
|
||||
}
|
||||
|
|
@ -361,6 +380,30 @@ public class LanguageInfo implements MemUsage.Instrumented {
|
|||
if (pos.startsWith("W")) return "W";
|
||||
return pos;
|
||||
}
|
||||
|
||||
// Uses a few rules to stem tokens
|
||||
public static String stem(String a) {
|
||||
int i = a.indexOf(' ');
|
||||
if (i != -1)
|
||||
return stem(a.substring(0, i)) + ' ' + stem(a.substring(i + 1));
|
||||
//Maybe we should just use the Stanford stemmer
|
||||
String res = a;
|
||||
//hard coded words
|
||||
if (a.equals("having") || a.equals("has")) res = "have";
|
||||
else if (a.equals("using")) res = "use";
|
||||
else if (a.equals("including")) res = "include";
|
||||
else if (a.equals("beginning")) res = "begin";
|
||||
else if (a.equals("utilizing")) res = "utilize";
|
||||
else if (a.equals("featuring")) res = "feature";
|
||||
else if (a.equals("preceding")) res = "precede";
|
||||
//rules
|
||||
else if (a.endsWith("ing")) res = a.substring(0, a.length() - 3);
|
||||
else if (a.endsWith("s") && !a.equals("'s")) res = a.substring(0, a.length() - 1);
|
||||
//don't return an empty string
|
||||
if (res.length() > 0) return res;
|
||||
return a;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -59,11 +59,11 @@ public class Learner {
|
|||
this.params = params;
|
||||
this.dataset = dataset;
|
||||
this.eventsOut = IOUtils.openOutAppendEasy(Execution.getFile("learner.events"));
|
||||
if (opts.initialization != null)
|
||||
if (opts.initialization != null && this.params.isEmpty())
|
||||
this.params.init(opts.initialization);
|
||||
// collect all semantic functions to update
|
||||
semFuncsToUpdate = new ArrayList<>();
|
||||
|
||||
// Collect all semantic functions to update.
|
||||
semFuncsToUpdate = new ArrayList<>();
|
||||
for (Rule rule : parser.grammar.getRules()) {
|
||||
SemanticFn currSemFn = rule.getSem();
|
||||
boolean toAdd = true;
|
||||
|
|
@ -125,11 +125,6 @@ public class Learner {
|
|||
Utils.systemHard("ln -sf params." + iter + " " + Execution.getFile("params"));
|
||||
}
|
||||
|
||||
// Write out examples and predictions
|
||||
if (opts.outputPredDerivations)
|
||||
for (String group : dataset.groups())
|
||||
ExampleUtils.writeSDF(iter, group, meanEvaluations.get(group), dataset.examples(group), opts.outputPredDerivations);
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
LogInfo.end_track();
|
||||
|
|
@ -199,6 +194,11 @@ public class Learner {
|
|||
if (opts.addFeedback && computeExpectedCounts)
|
||||
addFeedback(ex);
|
||||
|
||||
// Write out examples and predictions
|
||||
if (opts.outputPredDerivations && Builder.opts.parser.equals("FloatingParser")) {
|
||||
ExampleUtils.writeParaphraseSDF(iter, group, ex, opts.outputPredDerivations);
|
||||
}
|
||||
|
||||
// To save memory
|
||||
ex.predDerivations.clear();
|
||||
}
|
||||
|
|
@ -212,6 +212,7 @@ public class Learner {
|
|||
LogInfo.end_track();
|
||||
logEvaluationStats(evaluation, prefix);
|
||||
printLearnerEventsSummary(evaluation, iter, group);
|
||||
ExampleUtils.writeEvaluationSDF(iter, group, evaluation, examples.size());
|
||||
LogInfo.end_track();
|
||||
return evaluation;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ public class MarkFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof MarkFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -19,11 +19,16 @@ public class Master {
|
|||
public static class Options {
|
||||
@Option(gloss = "Execute these commands before starting")
|
||||
public List<String> scriptPaths = Lists.newArrayList();
|
||||
@Option(gloss = "Write out input lines to this path")
|
||||
@Option(gloss = "Execute these commands before starting (after scriptPaths)")
|
||||
public List<String> commands = Lists.newArrayList();
|
||||
@Option(gloss = "Write a log of this session to this path")
|
||||
public String logPath;
|
||||
|
||||
@Option(gloss = "Print help on startup")
|
||||
public boolean printHelp = true;
|
||||
|
||||
@Option(gloss = "Number of exchanges to keep in the context")
|
||||
public int contextMaxExchanges = 1;
|
||||
public int contextMaxExchanges = 0;
|
||||
|
||||
@Option(gloss = "Online update weights on new examples.")
|
||||
public boolean onlineLearnExamples = true;
|
||||
|
|
@ -93,6 +98,8 @@ public class Master {
|
|||
session = new Session(id);
|
||||
for (String path : opts.scriptPaths)
|
||||
processScript(session, path);
|
||||
for (String command : opts.commands)
|
||||
processQuery(session, command);
|
||||
if (id != null)
|
||||
sessions.put(id, session);
|
||||
}
|
||||
|
|
@ -116,13 +123,14 @@ public class Master {
|
|||
LogInfo.log(" (execute |logical form|): execute the logical form (e.g., (execute (call + (number 3) (number 4))))");
|
||||
LogInfo.log(" (def |key| |value|): define a macro to replace |key| with |value| in all commands (e.g., (def type fb:type.object type)))");
|
||||
LogInfo.log(" (context [(user |user|) (date |date|) (exchange |exchange|) (graph |graph|)]): prints out or set the context");
|
||||
LogInfo.log("Press Ctrl-D to exit.");
|
||||
}
|
||||
|
||||
public void runInteractivePrompt() {
|
||||
Session session = getSession("stdin");
|
||||
|
||||
printHelp();
|
||||
LogInfo.log("Press Ctrl-D to exit.");
|
||||
if (opts.printHelp)
|
||||
printHelp();
|
||||
|
||||
while (true) {
|
||||
LogInfo.stdout.print("> ");
|
||||
|
|
@ -215,12 +223,13 @@ public class Master {
|
|||
b.setContext(session.context);
|
||||
Example ex = b.createExample();
|
||||
|
||||
ex.preprocess(LanguageAnalyzer.getSingleton());
|
||||
ex.preprocess();
|
||||
|
||||
// Parse!
|
||||
builder.parser.parse(builder.params, ex, false);
|
||||
|
||||
response.ex = ex;
|
||||
ex.log();
|
||||
if (ex.predDerivations.size() > 0) {
|
||||
response.candidateIndex = 0;
|
||||
printDerivation(response.getDerivation());
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ public class MergeFormula extends Formula {
|
|||
return null;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof MergeFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -50,6 +50,11 @@ public class NaiveKnowledgeGraph extends KnowledgeGraph {
|
|||
tree.addChild(e2.toLispTree());
|
||||
return tree;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "<" + e1 + ", " + r + ", " + e2 + ">";
|
||||
}
|
||||
}
|
||||
|
||||
// Simplest graph representation: triples of values
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ public class NotFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof NotFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,13 @@ import fig.basic.*;
|
|||
* @author Percy Liang
|
||||
*/
|
||||
public class NumberFn extends SemanticFn {
|
||||
public static class Options {
|
||||
@Option(gloss = "Omit units") public boolean unitless = false;
|
||||
@Option(gloss = "Also test numbers by try converting to float (instead of using NER tags)")
|
||||
public boolean alsoTestByConversion = false;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
private List<String> requests; // List of types of fields to get (e.g., NUMBER)
|
||||
|
||||
private boolean request(String req) {
|
||||
|
|
@ -37,10 +44,10 @@ public class NumberFn extends SemanticFn {
|
|||
NumberValue numberValue = new NumberValue(Double.parseDouble(value));
|
||||
SemType type = numberValue.value == (int) numberValue.value ? SemType.intType : SemType.floatType;
|
||||
return new Derivation.Builder()
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
} catch (NumberFormatException e) {
|
||||
LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value);
|
||||
}
|
||||
|
|
@ -52,13 +59,15 @@ public class NumberFn extends SemanticFn {
|
|||
String value = ex.languageInfo.getNormalizedNerSpan("ORDINAL", c.getStart(), c.getEnd());
|
||||
if (value != null) {
|
||||
try {
|
||||
NumberValue numberValue = new NumberValue(Double.parseDouble(value), "fb:en.ordinal_number");
|
||||
NumberValue numberValue = (opts.unitless ?
|
||||
new NumberValue(Double.parseDouble(value)) :
|
||||
new NumberValue(Double.parseDouble(value), "fb:en.ordinal_number"));
|
||||
SemType type = SemType.intType;
|
||||
return new Derivation.Builder()
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
} catch (NumberFormatException e) {
|
||||
LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value);
|
||||
}
|
||||
|
|
@ -70,13 +79,15 @@ public class NumberFn extends SemanticFn {
|
|||
String value = ex.languageInfo.getNormalizedNerSpan("PERCENT", c.getStart(), c.getEnd());
|
||||
if (value != null) {
|
||||
try {
|
||||
NumberValue numberValue = new NumberValue(0.01 * Double.parseDouble(value.substring(1)));
|
||||
NumberValue numberValue = (opts.unitless ?
|
||||
new NumberValue(Double.parseDouble(value.substring(1))) :
|
||||
new NumberValue(0.01 * Double.parseDouble(value.substring(1))));
|
||||
SemType type = SemType.floatType;
|
||||
return new Derivation.Builder()
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
} catch (NumberFormatException e) {
|
||||
LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value);
|
||||
}
|
||||
|
|
@ -88,19 +99,39 @@ public class NumberFn extends SemanticFn {
|
|||
String value = ex.languageInfo.getNormalizedNerSpan("MONEY", c.getStart(), c.getEnd());
|
||||
if (value != null) {
|
||||
try {
|
||||
NumberValue numberValue = new NumberValue(Double.parseDouble(value.substring(1)), "fb:en.dollar");
|
||||
NumberValue numberValue = (opts.unitless ?
|
||||
new NumberValue(Double.parseDouble(value.substring(1))) :
|
||||
new NumberValue(Double.parseDouble(value.substring(1)), "fb:en.dollar"));
|
||||
SemType type = SemType.floatType;
|
||||
return new Derivation.Builder()
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
} catch (NumberFormatException e) {
|
||||
LogInfo.warnings("NumberFn: Cannot convert NerSpan \"%s\" to a number", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test by converting string to number directly (don't look at NER)
|
||||
if (opts.alsoTestByConversion && request("NUMBER") & c.getEnd() - c.getStart() == 1) {
|
||||
String value = ex.languageInfo.tokens.get(c.getStart());
|
||||
if (value != null) {
|
||||
try {
|
||||
NumberValue numberValue = new NumberValue(Double.parseDouble(value));
|
||||
SemType type = numberValue.value == (int) numberValue.value ? SemType.intType : SemType.floatType;
|
||||
return new Derivation.Builder()
|
||||
.withCallable(c)
|
||||
.formula(new ValueFormula<>(numberValue))
|
||||
.type(type)
|
||||
.createDerivation();
|
||||
} catch (NumberFormatException e) {
|
||||
// Don't issue warnings; most spans are not numbers
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ public class Params {
|
|||
@Option(gloss = "Use dual averaging") public boolean dualAveraging = false;
|
||||
@Option(gloss = "Whether to do lazy l1 reg updates") public String l1Reg = "none";
|
||||
@Option(gloss = "L1 reg coefficient") public double l1RegCoeff = 0d;
|
||||
@Option(gloss = "Lazy L1 full update frequency") public int lazyL1FullUpdateFreq = 5000;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
public enum L1Reg {
|
||||
|
|
@ -49,7 +50,7 @@ public class Params {
|
|||
private L1Reg l1Reg = parseReg(opts.l1Reg);
|
||||
|
||||
// Discriminative weights
|
||||
private HashMap<String, Double> weights = new HashMap<>();
|
||||
private Map<String, Double> weights = new HashMap<>();
|
||||
|
||||
// For AdaGrad
|
||||
Map<String, Double> sumSquaredGradients = new HashMap<>();
|
||||
|
|
@ -110,13 +111,12 @@ public class Params {
|
|||
|
||||
// Update weights by adding |gradient| (modified appropriately with step size).
|
||||
public synchronized void update(Map<String, Double> gradient) {
|
||||
numUpdates++;
|
||||
|
||||
for (Map.Entry<String, Double> entry : gradient.entrySet()) {
|
||||
String f = entry.getKey();
|
||||
double g = entry.getValue();
|
||||
if (g * g == 0) continue; // In order to not divide by zero
|
||||
|
||||
if (l1Reg == L1Reg.LAZY) lazyL1Update(f);
|
||||
double stepSize = computeStepSize(f, g);
|
||||
|
||||
if (opts.dualAveraging) {
|
||||
|
|
@ -132,17 +132,25 @@ public class Params {
|
|||
throw new RuntimeException("Gradient absolute value is too large or too small");
|
||||
}
|
||||
MapUtils.incr(weights, f, stepSize * g);
|
||||
if (l1Reg == L1Reg.LAZY) l1UpdateTimeMap.put(f, numUpdates);
|
||||
}
|
||||
}
|
||||
// non lazy implementation goes over all weights
|
||||
if (l1Reg == L1Reg.NONLAZY) {
|
||||
Set<String> features = new HashSet<String>(weights.keySet());
|
||||
for (String f :features) {
|
||||
for (String f : features) {
|
||||
double stepSize = computeStepSize(f, 0d); // no update for gradient here
|
||||
double update = opts.l1RegCoeff * -Math.signum(MapUtils.getDouble(weights, f, opts.defaultWeight));
|
||||
clipUpdate(f, stepSize * update);
|
||||
}
|
||||
}
|
||||
numUpdates++;
|
||||
if (l1Reg == L1Reg.LAZY && opts.lazyL1FullUpdateFreq > 0 && numUpdates % opts.lazyL1FullUpdateFreq == 0) {
|
||||
LogInfo.begin_track("Fully apply L1 regularization.");
|
||||
finalizeWeights();
|
||||
System.gc();
|
||||
LogInfo.end_track();
|
||||
}
|
||||
}
|
||||
|
||||
private double computeStepSize(String feature, double gradient) {
|
||||
|
|
@ -174,20 +182,24 @@ public class Params {
|
|||
}
|
||||
|
||||
private void lazyL1Update(String f) {
|
||||
if (MapUtils.getDouble(weights, f, 0.0) == 0) return;
|
||||
// For pre-initialized weights, which have no updates yet
|
||||
if (sumSquaredGradients.get(f) == null || l1UpdateTimeMap.get(f) == null) {
|
||||
l1UpdateTimeMap.put(f, numUpdates);
|
||||
sumSquaredGradients.put(f, 0.0);
|
||||
return;
|
||||
}
|
||||
int numOfIter = numUpdates - l1UpdateTimeMap.get(f);
|
||||
if (numOfIter <= 0) {
|
||||
l1UpdateTimeMap.put(f, numUpdates);
|
||||
return;
|
||||
}
|
||||
int numOfIter = numUpdates - MapUtils.get(l1UpdateTimeMap, f, 0);
|
||||
if (numOfIter == 0) return;
|
||||
if (numOfIter < 0) throw new RuntimeException("l1UpdateTimeMap is out of sync.");
|
||||
|
||||
double stepSize = (numOfIter * opts.initStepSize) / (Math.sqrt(sumSquaredGradients.get(f) + 1));
|
||||
double update = -opts.l1RegCoeff * Math.signum(MapUtils.getDouble(weights, f, 0.0));
|
||||
clipUpdate(f, stepSize * update);
|
||||
l1UpdateTimeMap.put(f, numUpdates);
|
||||
if (weights.containsKey(f))
|
||||
l1UpdateTimeMap.put(f, numUpdates);
|
||||
else
|
||||
l1UpdateTimeMap.remove(f);
|
||||
}
|
||||
|
||||
public synchronized double getWeight(String f) {
|
||||
|
|
@ -200,7 +212,7 @@ public class Params {
|
|||
}
|
||||
}
|
||||
|
||||
public synchronized Map<String, Double> getWeights() { return weights; }
|
||||
public synchronized Map<String, Double> getWeights() { finalizeWeights(); return weights; }
|
||||
|
||||
public void write(PrintWriter out) { write(null, out); }
|
||||
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import fig.basic.IOUtils;
|
||||
import fig.basic.Option;
|
||||
import fig.basic.MapUtils;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
/**
|
||||
* ParaphraseModel extracts and scores paraphrasing featues from derivations.
|
||||
* This model is intended to be used with FloatingParser
|
||||
*
|
||||
* @author Yushi Wang
|
||||
*/
|
||||
|
||||
public final class ParaphraseModel {
|
||||
public static class Options {
|
||||
@Option(gloss = "Path to file with alignment table")
|
||||
public String paraphraseModelPath = "regex/regex-alignments.txt";
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
||||
public static ParaphraseModel model;
|
||||
|
||||
Map<String, Map<String, Double>> table;
|
||||
|
||||
// We should only have one paraphrase model
|
||||
public static ParaphraseModel getSingleton() {
|
||||
if (model == null) {
|
||||
model = new ParaphraseModel();
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
private ParaphraseModel() {
|
||||
table = loadParaphraseModel(opts.paraphraseModelPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loading paraphrase model from file
|
||||
*/
|
||||
private Map<String, Map<String, Double>> loadParaphraseModel(String path) {
|
||||
LogInfo.begin_track("Loading paraphrase model");
|
||||
Map<String, Map<String, Double>> res = new HashMap<String, Map<String, Double>>();
|
||||
for (String line: IOUtils.readLinesHard(path)) {
|
||||
String[] tokens = line.split("\t");
|
||||
// double count = Double.parseDouble(tokens[2]);
|
||||
MapUtils.putIfAbsent(res, tokens[0], new HashMap<String, Double>());
|
||||
for (String token : tokens)
|
||||
LogInfo.logs("%s", token);
|
||||
res.get(tokens[0]).put(tokens[1], 1.0);
|
||||
/* Alignment stats don't matter right now
|
||||
if(count>=threshold) {
|
||||
AlignmentStats aStats = new AlignmentStats(count, Double.parseDouble(tokens[3]), Double.parseDouble(tokens[4]));
|
||||
res.get(tokens[0]).put(tokens[1], aStats);
|
||||
}
|
||||
*/
|
||||
}
|
||||
LogInfo.logs("ParaphraseUtils.loadPhraseTable: number of entries=%s", res.size());
|
||||
LogInfo.end_track();
|
||||
return res;
|
||||
}
|
||||
|
||||
public boolean containsKey(String key) {
|
||||
return table.containsKey(key);
|
||||
}
|
||||
|
||||
public Double get(String key, String token) {
|
||||
if (!table.containsKey(key) || !table.get(key).containsKey(token)) return 0.0;
|
||||
return table.get(key).get(token);
|
||||
}
|
||||
}
|
||||
|
|
@ -21,6 +21,8 @@ public abstract class Parser {
|
|||
|
||||
@Option(gloss = "Maximal number of predictions to print")
|
||||
public int maxPrintedPredictions = Integer.MAX_VALUE;
|
||||
@Option(gloss = "Maximal number of correct predictions to print")
|
||||
public int maxPrintedTrue = Integer.MAX_VALUE;
|
||||
|
||||
@Option(gloss = "Use a coarse pass to prune the chart before full parsing")
|
||||
public boolean coarsePrune = false;
|
||||
|
|
@ -42,6 +44,18 @@ public abstract class Parser {
|
|||
|
||||
@Option(gloss = "Whether to unroll derivation streams (applies to lazy parsers)")
|
||||
public boolean unrollStream = false;
|
||||
|
||||
@Option(gloss = "Inject random noise into the score (to mix things up a bit)")
|
||||
public double derivationScoreNoise = 0;
|
||||
|
||||
@Option(gloss = "Source of random noise")
|
||||
public Random derivationScoreRandom = new Random(1);
|
||||
|
||||
@Option (gloss = "Prune away error denotations")
|
||||
public boolean pruneErrorValues = false;
|
||||
|
||||
@Option(gloss = "Dump all features (for debugging)")
|
||||
public boolean dumpAllFeatures = false;
|
||||
}
|
||||
|
||||
public static final Options opts = new Options();
|
||||
|
|
@ -155,6 +169,11 @@ public abstract class Parser {
|
|||
ex.evaluation = new Evaluation();
|
||||
addToEvaluation(state, ex.evaluation);
|
||||
|
||||
// Clean up temporary state used during parsing
|
||||
ex.clearTempState();
|
||||
for (Derivation deriv : ex.predDerivations)
|
||||
deriv.clearTempState();
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +233,6 @@ public abstract class Parser {
|
|||
double topMass = 0;
|
||||
if (ex.targetValue != null) {
|
||||
while (numTop < numCandidates &&
|
||||
compatibilities[numTop] > 0.0d &&
|
||||
Math.abs(predDerivations.get(numTop).score - predDerivations.get(0).score) < 1e-10) {
|
||||
topMass += probs[numTop];
|
||||
numTop++;
|
||||
|
|
@ -248,21 +266,33 @@ public abstract class Parser {
|
|||
}
|
||||
|
||||
// Fully correct
|
||||
int numPrintedSoFar = 0;
|
||||
for (int i = 0; i < predDerivations.size(); i++) {
|
||||
Derivation deriv = predDerivations.get(i);
|
||||
if (compatibilities != null && compatibilities[i] == 1) {
|
||||
LogInfo.logs(
|
||||
"True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
|
||||
Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
|
||||
boolean print = printAllPredictions || (numPrintedSoFar < opts.maxPrintedTrue);
|
||||
if (print) {
|
||||
LogInfo.logs(
|
||||
"True@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
|
||||
Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
|
||||
numPrintedSoFar++;
|
||||
if (opts.dumpAllFeatures) FeatureVector.logFeatureWeights("Features", deriv.getAllFeatureVector(), state.params);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Partially correct
|
||||
numPrintedSoFar = 0;
|
||||
for (int i = 0; i < predDerivations.size(); i++) {
|
||||
Derivation deriv = predDerivations.get(i);
|
||||
if (compatibilities != null && compatibilities[i] > 0 && compatibilities[i] < 1) {
|
||||
LogInfo.logs(
|
||||
"Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
|
||||
Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
|
||||
boolean print = printAllPredictions || (numPrintedSoFar < opts.maxPrintedTrue);
|
||||
if (print) {
|
||||
LogInfo.logs(
|
||||
"Part@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
|
||||
Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
|
||||
numPrintedSoFar++;
|
||||
if (opts.dumpAllFeatures) FeatureVector.logFeatureWeights("Features", deriv.getAllFeatureVector(), state.params);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Anything that's predicted.
|
||||
|
|
@ -276,6 +306,7 @@ public abstract class Parser {
|
|||
"Pred@%04d: %s [score=%s, prob=%s%s]", i, deriv.toString(),
|
||||
Fmt.D(deriv.score), Fmt.D(probs[i]), compatibilities != null ? ", comp=" + Fmt.D(compatibilities[i]) : "");
|
||||
// LogInfo.logs("Derivation tree: %s", deriv.toRecursiveString());
|
||||
if (opts.dumpAllFeatures) FeatureVector.logFeatureWeights("Features", deriv.getAllFeatureVector(), state.params);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import fig.basic.Fmt;
|
|||
import fig.basic.LogInfo;
|
||||
import fig.basic.NumUtils;
|
||||
import fig.basic.Evaluation;
|
||||
import fig.basic.Option;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
|
@ -16,6 +17,14 @@ import java.util.Map;
|
|||
* @author Percy Liang
|
||||
*/
|
||||
public abstract class ParserState {
|
||||
public static class Options {
|
||||
@Option(gloss = "Use a custom distribution for computing expected counts")
|
||||
public CustomExpectedCount customExpectedCounts = CustomExpectedCount.NONE;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
public enum CustomExpectedCount { NONE, UNIFORM, TOP, RANDOM, }
|
||||
|
||||
//// Input: specification of how to parse
|
||||
|
||||
public final Parser parser;
|
||||
|
|
@ -41,8 +50,6 @@ public abstract class ParserState {
|
|||
public int totalGeneratedDerivs; // Total number of derivations produced
|
||||
public int numOfFeaturizedDerivs = 0; // Number of derivations featured
|
||||
|
||||
|
||||
@SuppressWarnings({ "unchecked" })
|
||||
public ParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) {
|
||||
this.parser = parser;
|
||||
this.params = params;
|
||||
|
|
@ -70,7 +77,7 @@ public abstract class ParserState {
|
|||
|
||||
if (parser.verbose(5)) {
|
||||
LogInfo.logs("featurizeAndScoreDerivation(score=%s) %s %s: %s [rule: %s]",
|
||||
Fmt.D(deriv.score), deriv.cat, ex.spanString(deriv.start, deriv.end), deriv, deriv.rule);
|
||||
Fmt.D(deriv.score), deriv.cat, ex.spanString(deriv.start, deriv.end), deriv, deriv.rule);
|
||||
}
|
||||
numOfFeaturizedDerivs++;
|
||||
}
|
||||
|
|
@ -113,8 +120,26 @@ public abstract class ParserState {
|
|||
i++;
|
||||
}
|
||||
|
||||
// Inject noise into the noise (to simulate sampling); ideally would add Gumbel noise
|
||||
if (Parser.opts.derivationScoreNoise > 0) {
|
||||
for (Derivation deriv : derivations)
|
||||
deriv.score += Parser.opts.derivationScoreRandom.nextDouble() * Parser.opts.derivationScoreNoise;
|
||||
}
|
||||
|
||||
Derivation.sortByScore(derivations);
|
||||
|
||||
// Print out information
|
||||
if (parser.opts.verbose >= 3) {
|
||||
LogInfo.begin_track("ParserState.pruneCell(%s): %d derivations", cellDescription, derivations.size());
|
||||
for (Derivation deriv : derivations) {
|
||||
LogInfo.logs("%s(%s,%s): %s %s, [score=%s]", deriv.cat, deriv.start, deriv.end, deriv.formula,
|
||||
deriv.canonicalUtterance, deriv.score);
|
||||
}
|
||||
|
||||
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
// Max beam position (after sorting)
|
||||
i = 0;
|
||||
for (Derivation deriv : derivations) {
|
||||
|
|
@ -142,46 +167,46 @@ public abstract class ParserState {
|
|||
// All tokens (length 1)
|
||||
for (int i = 0; i < numTokens; i++) {
|
||||
derivs.add(
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.tokenCat).start(i).end(i + 1)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.token(i))
|
||||
.canonicalUtterance(ex.token(i))
|
||||
.createDerivation());
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.tokenCat).start(i).end(i + 1)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.token(i))
|
||||
.canonicalUtterance(ex.token(i))
|
||||
.createDerivation());
|
||||
|
||||
// Lemmatized version
|
||||
derivs.add(
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.lemmaTokenCat).start(i).end(i + 1)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.lemmaToken(i))
|
||||
.canonicalUtterance(ex.token(i))
|
||||
.createDerivation());
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.lemmaTokenCat).start(i).end(i + 1)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.lemmaToken(i))
|
||||
.canonicalUtterance(ex.token(i))
|
||||
.createDerivation());
|
||||
}
|
||||
|
||||
// All phrases (any length)
|
||||
for (int i = 0; i < numTokens; i++) {
|
||||
for (int j = i + 1; j <= numTokens; j++) {
|
||||
derivs.add(
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.phraseCat).start(i).end(j)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.phrase(i, j))
|
||||
.canonicalUtterance(ex.phrase(i, j))
|
||||
.createDerivation());
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.phraseCat).start(i).end(j)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.phrase(i, j))
|
||||
.canonicalUtterance(ex.phrase(i, j))
|
||||
.createDerivation());
|
||||
|
||||
// Lemmatized version
|
||||
derivs.add(
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.lemmaPhraseCat).start(i).end(j)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.lemmaPhrase(i, j))
|
||||
.canonicalUtterance(ex.phrase(i, j))
|
||||
.createDerivation());
|
||||
new Derivation.Builder()
|
||||
.cat(Rule.lemmaPhraseCat).start(i).end(j)
|
||||
.rule(Rule.nullRule)
|
||||
.children(Derivation.emptyList)
|
||||
.withStringFormulaFrom(ex.lemmaPhrase(i, j))
|
||||
.canonicalUtterance(ex.phrase(i, j))
|
||||
.createDerivation());
|
||||
}
|
||||
}
|
||||
return derivs;
|
||||
|
|
@ -231,12 +256,36 @@ public abstract class ParserState {
|
|||
|
||||
trueScores = new double[n];
|
||||
predScores = new double[n];
|
||||
// For update schemas that choose one good and one bad candidate to update
|
||||
int[] goodAndBad = null;
|
||||
if (opts.customExpectedCounts == CustomExpectedCount.TOP) {
|
||||
goodAndBad = getTopDerivations(derivations);
|
||||
if (goodAndBad == null) return;
|
||||
} else if (opts.customExpectedCounts == CustomExpectedCount.RANDOM) {
|
||||
goodAndBad = getRandomDerivations(derivations);
|
||||
if (goodAndBad == null) return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
Derivation deriv = derivations.get(i);
|
||||
double logReward = Math.log(compatibilityToReward(deriv.compatibility));
|
||||
|
||||
trueScores[i] = deriv.score + logReward;
|
||||
predScores[i] = deriv.score;
|
||||
switch (opts.customExpectedCounts) {
|
||||
case NONE:
|
||||
trueScores[i] = deriv.score + logReward;
|
||||
predScores[i] = deriv.score;
|
||||
break;
|
||||
case UNIFORM:
|
||||
trueScores[i] = logReward;
|
||||
predScores[i] = 0;
|
||||
break;
|
||||
case TOP: case RANDOM:
|
||||
trueScores[i] = (i == goodAndBad[0]) ? 0 : Double.NEGATIVE_INFINITY;
|
||||
predScores[i] = (i == goodAndBad[1]) ? 0 : Double.NEGATIVE_INFINITY;
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Unknown customExpectedCounts: " + opts.customExpectedCounts);
|
||||
}
|
||||
}
|
||||
|
||||
// Usually this happens when there are no derivations.
|
||||
|
|
@ -247,7 +296,45 @@ public abstract class ParserState {
|
|||
for (int i = 0; i < n; i++) {
|
||||
Derivation deriv = derivations.get(i);
|
||||
double incr = trueScores[i] - predScores[i];
|
||||
if (incr == 0) continue;
|
||||
deriv.incrementAllFeatureVector(incr, counts);
|
||||
}
|
||||
}
|
||||
|
||||
private static int[] getTopDerivations(List<Derivation> derivations) {
|
||||
int chosenGood = -1, chosenBad = -1;
|
||||
double chosenGoodScore = Double.NEGATIVE_INFINITY, chosenBadScore = Double.NEGATIVE_INFINITY;
|
||||
for (int i = 0; i < derivations.size(); i++) {
|
||||
Derivation deriv = derivations.get(i);
|
||||
if (deriv.compatibility == 1) { // good
|
||||
if (deriv.score > chosenGoodScore) {
|
||||
chosenGood = i; chosenGoodScore = deriv.score;
|
||||
}
|
||||
} else { // bad
|
||||
if (deriv.score > chosenBadScore) {
|
||||
chosenBad = i; chosenBadScore = deriv.score;
|
||||
}
|
||||
}
|
||||
}
|
||||
return (chosenGood == -1 || chosenBad == -1) ? null : new int[] {chosenGood, chosenBad};
|
||||
}
|
||||
|
||||
private static int[] getRandomDerivations(List<Derivation> derivations) {
|
||||
int chosenGood = -1, chosenBad = -1, numGoodSoFar = 0, numBadSoFar = 0;
|
||||
for (int i = 0; i < derivations.size(); i++) {
|
||||
Derivation deriv = derivations.get(i);
|
||||
if (deriv.compatibility == 1) {
|
||||
numGoodSoFar++;
|
||||
if (Math.random() <= 1.0 / numGoodSoFar) {
|
||||
chosenGood = i;
|
||||
}
|
||||
} else { // bad
|
||||
numBadSoFar++;
|
||||
if (Math.random() <= 1.0 / numBadSoFar) {
|
||||
chosenBad = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
return (chosenGood == -1 || chosenBad == -1) ? null : new int[] {chosenGood, chosenBad};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -412,7 +412,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
|
|||
updateBackpointers(pds.derivStream, nextDeriv);
|
||||
|
||||
DerivationStream derivStream = SingleDerivationStream.constant(nextDeriv);
|
||||
if (parser.verbose(3)) {
|
||||
if (parser.verbose(3) && derivStream.hasNext()) {
|
||||
Derivation deriv = derivStream.peek();
|
||||
LogInfo.logs("unrollHighProbStreams(): add deriv=%s(%s,%s) [%s] score=%s, |stream|=%s",
|
||||
deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.score, pds.derivStream.estimatedSize());
|
||||
|
|
@ -820,7 +820,7 @@ final class ReinforcementParserState extends AbstractReinforcementParserState {
|
|||
modified = true;
|
||||
Derivation nextDeriv = pds.derivStream.next();
|
||||
DerivationStream newDerivStream = SingleDerivationStream.constant(nextDeriv);
|
||||
if (parser.verbose(3)) {
|
||||
if (parser.verbose(3) && newDerivStream.hasNext()) {
|
||||
Derivation deriv = newDerivStream.peek();
|
||||
LogInfo.logs("MultiplicativeSampler.unroll(): add necessary deriv=%s(%s,%s) [%s] score=%s, |stream|=%s, creationIndex=%s",
|
||||
deriv.cat, deriv.start, deriv.end, deriv.formula, deriv.score, pds.derivStream.estimatedSize(), deriv.creationIndex);
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ public class ReverseFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof ReverseFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -75,6 +75,16 @@ public class Rule {
|
|||
return true;
|
||||
}
|
||||
|
||||
// Return the number of categories on the RHS
|
||||
public int numRhsCats() {
|
||||
int ret = 0;
|
||||
for (int i = 0; i < rhs.size(); ++i) {
|
||||
if (isCat(rhs.get(i)))
|
||||
ret++;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public LispTree toLispTree() {
|
||||
LispTree tree = LispTree.proto.newList();
|
||||
tree.addChild("rule");
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ public abstract class SemType {
|
|||
public static final SemType intType = new AtomicSemType(CanonicalNames.INT);
|
||||
public static final SemType floatType = new AtomicSemType(CanonicalNames.FLOAT);
|
||||
public static final SemType dateType = new AtomicSemType(CanonicalNames.DATE);
|
||||
public static final SemType timeType = new AtomicSemType(CanonicalNames.TIME);
|
||||
public static final SemType numberType = new AtomicSemType(CanonicalNames.NUMBER);
|
||||
public static final SemType numberOrDateType = new UnionSemType(numberType, dateType);
|
||||
public static final SemType entityType = new AtomicSemType(CanonicalNames.ENTITY);
|
||||
|
|
|
|||
|
|
@ -38,18 +38,36 @@ public class SimpleAnalyzer extends LanguageAnalyzer {
|
|||
utterance = breakHyphens(utterance);
|
||||
|
||||
// Default analysis - create tokens crudely
|
||||
utterance = utterance.replaceAll(",", " ,");
|
||||
utterance = utterance.replaceAll("\\.", " .");
|
||||
utterance = utterance.replaceAll("\\?", " ?");
|
||||
utterance = utterance.replaceAll("'", " '");
|
||||
utterance = utterance.replaceAll("\\[", " [ ");
|
||||
utterance = utterance.replaceAll("\\]", " ] ");
|
||||
utterance = utterance.trim();
|
||||
StringBuilder buf = new StringBuilder();
|
||||
for (int i = 0; i < utterance.length(); i++) {
|
||||
char c = utterance.charAt(i);
|
||||
// Put whitespace around certain characters.
|
||||
// TODO(pliang): handle contractions such as "can't" properly.
|
||||
boolean boundaryBefore = !(i - 1 >= 0) || utterance.charAt(i - 1) == ' ';
|
||||
boolean boundaryAfter = !(i + 1 < utterance.length()) || utterance.charAt(i + 1) == ' ';
|
||||
boolean separate;
|
||||
if (c == '.') // Break off period if already space around it (to preserve numbers like 3.5)
|
||||
separate = boundaryBefore || boundaryAfter;
|
||||
else
|
||||
separate = (",?'\"[]".indexOf(c) != -1);
|
||||
|
||||
if (separate) buf.append(' ');
|
||||
// Convert quotes
|
||||
if (c == '"')
|
||||
buf.append(boundaryBefore ? "``" : "''");
|
||||
else if (c == '\'')
|
||||
buf.append(boundaryBefore ? "`" : "'");
|
||||
else
|
||||
buf.append(c);
|
||||
if (separate) buf.append(' ');
|
||||
}
|
||||
utterance = buf.toString().trim();
|
||||
if (!utterance.equals("")) {
|
||||
for (String token : utterance.split("\\s+")) {
|
||||
String[] tokens = utterance.split("\\s+");
|
||||
for (String token : tokens) {
|
||||
languageInfo.tokens.add(LanguageAnalyzer.opts.lowerCaseTokens ? token.toLowerCase() : token);
|
||||
String lemma = token;
|
||||
if (token.endsWith("s"))
|
||||
if (token.endsWith("s") && token.length() > 1)
|
||||
lemma = token.substring(0, token.length() - 1);
|
||||
languageInfo.lemmaTokens.add(LanguageAnalyzer.opts.lowerCaseTokens ? lemma.toLowerCase() : lemma);
|
||||
|
||||
|
|
@ -65,15 +83,14 @@ public class SimpleAnalyzer extends LanguageAnalyzer {
|
|||
try {
|
||||
Double.parseDouble(token);
|
||||
languageInfo.posTags.add("CD");
|
||||
if (token.length() == 4)
|
||||
languageInfo.nerTags.add("DATE");
|
||||
else
|
||||
languageInfo.nerTags.add("NUMBER");
|
||||
languageInfo.nerTags.add("NUMBER");
|
||||
languageInfo.nerValues.add(token);
|
||||
} catch (NumberFormatException e) {
|
||||
// Guess that capitalized nouns are proper
|
||||
if (Character.isUpperCase(token.charAt(0)))
|
||||
languageInfo.posTags.add("NNP");
|
||||
else if (token.equals("'") || token.equals("\"") || token.equals("''") || token.equals("``"))
|
||||
languageInfo.posTags.add("''");
|
||||
else
|
||||
languageInfo.posTags.add("UNK");
|
||||
languageInfo.nerTags.add("UNK");
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ public class SuperlativeFormula extends Formula {
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof SuperlativeFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import fig.basic.*;
|
||||
|
||||
/**
|
||||
* Preprocess the targetValue of an example.
|
||||
*
|
||||
* @author ppasupat
|
||||
*/
|
||||
public abstract class TargetValuePreprocessor {
|
||||
public static class Options {
|
||||
@Option public String targetValuePreprocessor = null;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
private static TargetValuePreprocessor singleton;
|
||||
|
||||
public static TargetValuePreprocessor getSingleton() {
|
||||
if (singleton == null) {
|
||||
if (opts.targetValuePreprocessor == null || opts.targetValuePreprocessor.isEmpty())
|
||||
singleton = new IdentityTargetValuePreprocessor();
|
||||
else
|
||||
singleton = (TargetValuePreprocessor) Utils.newInstanceHard(
|
||||
SempreUtils.resolveClassName(opts.targetValuePreprocessor));
|
||||
}
|
||||
return singleton;
|
||||
}
|
||||
public static void setSingleton(TargetValuePreprocessor processor) { singleton = processor; }
|
||||
|
||||
public abstract Value preprocess(Value value);
|
||||
|
||||
}
|
||||
|
||||
class IdentityTargetValuePreprocessor extends TargetValuePreprocessor {
|
||||
public Value preprocess(Value value) { return value; }
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import fig.basic.LispTree;
|
||||
|
||||
/**
|
||||
* Created by joberant on 1/23/15.
|
||||
* Value for representing time
|
||||
*/
|
||||
public class TimeValue extends Value {
|
||||
|
||||
public final int hour;
|
||||
public final int minute;
|
||||
|
||||
public TimeValue(int hour, int minute) {
|
||||
if (hour > 23 || hour < 0) throw new RuntimeException("Illegal hour: " + hour);
|
||||
if (minute > 59 || minute < 0) throw new RuntimeException("Illegal minute: " + minute);
|
||||
this.hour = hour;
|
||||
this.minute = minute;
|
||||
}
|
||||
|
||||
public TimeValue(LispTree tree) {
|
||||
this.hour = Integer.valueOf(tree.child(1).value);
|
||||
this.minute = Integer.valueOf(tree.child(2).value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public LispTree toLispTree() {
|
||||
LispTree tree = LispTree.proto.newList();
|
||||
tree.addChild("time");
|
||||
tree.addChild(String.valueOf(hour));
|
||||
tree.addChild(String.valueOf(minute));
|
||||
return tree;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
TimeValue timeValue = (TimeValue) o;
|
||||
if (hour != timeValue.hour) return false;
|
||||
if (minute != timeValue.minute) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = hour;
|
||||
result = 31 * result + minute;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
package edu.stanford.nlp.sempre;
|
||||
|
||||
import fig.basic.*;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
|
|
@ -56,6 +57,13 @@ public final class TypeInference {
|
|||
}
|
||||
|
||||
private static final ValueFormula typeFormula = new ValueFormula(new NameValue(CanonicalNames.TYPE));
|
||||
|
||||
private static final Set<Formula> comparisonFormulas = new HashSet<>(Arrays.asList(
|
||||
new ValueFormula(new NameValue("<")),
|
||||
new ValueFormula(new NameValue(">")),
|
||||
new ValueFormula(new NameValue("<=")),
|
||||
new ValueFormula(new NameValue(">="))));
|
||||
|
||||
private static class TypeException extends Exception { }
|
||||
|
||||
private static class Env {
|
||||
|
|
@ -109,16 +117,6 @@ public final class TypeInference {
|
|||
return type;
|
||||
}
|
||||
|
||||
// Unary: fb:domain.type [contains exactly one period]
|
||||
// Binary: fb:domain.type.property, <, >, etc.
|
||||
private static boolean isUnary(String s) {
|
||||
int i = s.indexOf('.');
|
||||
if (i == -1) return false;
|
||||
i = s.indexOf('.', i + 1);
|
||||
if (i == -1) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static SemType check(SemType type) throws TypeException {
|
||||
if (!type.isValid()) throw new TypeException();
|
||||
return type;
|
||||
|
|
@ -137,10 +135,11 @@ public final class TypeInference {
|
|||
if (value instanceof NumberValue) return check(type.meet(SemType.numberType));
|
||||
else if (value instanceof StringValue) return check(type.meet(SemType.stringType));
|
||||
else if (value instanceof DateValue) return check(type.meet(SemType.dateType));
|
||||
else if (value instanceof TimeValue) return check(type.meet(SemType.timeType));
|
||||
else if (value instanceof NameValue) {
|
||||
String id = ((NameValue) value).id;
|
||||
|
||||
if (isUnary(id)) { // Unary
|
||||
if (CanonicalNames.isUnary(id)) { // Unary
|
||||
SemType unaryType = env.typeLookup.getEntityType(id);
|
||||
if (unaryType == null)
|
||||
unaryType = SemType.entityType;
|
||||
|
|
@ -170,6 +169,10 @@ public final class TypeInference {
|
|||
if (typeFormula.equals(join.relation) && join.child instanceof ValueFormula)
|
||||
return check(type.meet(SemType.newAtomicSemType(Formulas.getString(join.child))));
|
||||
|
||||
// Special case: (<= (number 5)) => same type as (number 5)
|
||||
if (comparisonFormulas.contains(join.relation))
|
||||
return check(type.meet(inferType(join.child, env, SemType.numberOrDateType)));
|
||||
|
||||
SemType relationType = inferType(join.relation, env, new FuncSemType(SemType.topType, type)); // Relation
|
||||
SemType childType = inferType(join.child, env, relationType.getArgType()); // Child
|
||||
relationType = inferType(join.relation, env, new FuncSemType(childType, type)); // Relation again
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonValue;
|
|||
import fig.basic.LispTree;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
import java.util.Comparator;
|
||||
|
||||
/**
|
||||
* Values represent denotations (or partial denotations).
|
||||
*
|
||||
|
|
@ -26,4 +28,11 @@ public abstract class Value {
|
|||
|
||||
@Override public abstract boolean equals(Object o);
|
||||
@Override public abstract int hashCode();
|
||||
|
||||
public static class ValueComparator implements Comparator<Value> {
|
||||
@Override
|
||||
public int compare(Value o1, Value o2) {
|
||||
return o1.toString().compareTo(o2.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ public class ValueFormula<T extends Value> extends PrimitiveFormula {
|
|||
return value.toLispTree();
|
||||
}
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ public final class Values {
|
|||
if ("context".equals(type)) return new ContextValue(tree);
|
||||
if ("date".equals(type)) return new DateValue(tree);
|
||||
if ("error".equals(type)) return new ErrorValue(tree);
|
||||
if ("time".equals(type)) return new TimeValue(tree);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ public class VariableFormula extends PrimitiveFormula {
|
|||
public VariableFormula(String name) { this.name = name; }
|
||||
public LispTree toLispTree() { return LispTree.proto.newList("var", name); }
|
||||
|
||||
@SuppressWarnings({"equalshashcode"})
|
||||
@Override
|
||||
public boolean equals(Object thatObj) {
|
||||
if (!(thatObj instanceof VariableFormula)) return false;
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
<project>
|
||||
<target name="compile">
|
||||
<property name="module_name" value="core"/>
|
||||
<property name="root" location="../../../../../"/>
|
||||
<!--<property name="classes" location="${root}/classes/${module_name}"/>-->
|
||||
<property name="classes" location="${root}/classes"/>
|
||||
<mkdir dir="${classes}"/>
|
||||
<mkdir dir="${root}/libsempre"/>
|
||||
<path id="lib.path"><fileset dir="${root}/lib"/><fileset dir="${root}/libsempre"/></path>
|
||||
<javac srcdir="${root}/src" destdir="${classes}" classpathref="lib.path" debug="true" includeantruntime="false">
|
||||
<include name="edu/stanford/nlp/sempre/*.java"/>
|
||||
<include name="edu/stanford/nlp/sempre/test/"/>
|
||||
</javac>
|
||||
<jar destfile="${root}/libsempre/sempre-${module_name}.jar" basedir="${classes}" includes="edu/**">
|
||||
</jar>
|
||||
</target>
|
||||
</project>
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
<project>
|
||||
<target name="compile">
|
||||
<property name="module_name" value="cache"/>
|
||||
<property name="root" location="../../../../../../"/>
|
||||
<property name="classes" location="${root}/classes/${module_name}"/>
|
||||
<mkdir dir="${classes}"/>
|
||||
<mkdir dir="${root}/libsempre"/>
|
||||
<path id="lib.path"><fileset dir="${root}/lib"/><fileset dir="${root}/libsempre"/></path>
|
||||
<javac srcdir="${root}/src" destdir="${classes}" classpathref="lib.path" debug="true" includeantruntime="false">
|
||||
<include name="edu/stanford/nlp/sempre/cache/"/>
|
||||
</javac>
|
||||
<jar destfile="${root}/libsempre/sempre-${module_name}.jar" basedir="${classes}"/>
|
||||
</target>
|
||||
</project>
|
||||
|
|
@ -2,7 +2,6 @@ package edu.stanford.nlp.sempre.corenlp;
|
|||
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
import edu.stanford.nlp.sempre.LanguageInfo.DependencyEdge;
|
||||
|
||||
import edu.stanford.nlp.ling.CoreAnnotations;
|
||||
import edu.stanford.nlp.ling.IndexedWord;
|
||||
import edu.stanford.nlp.ling.CoreAnnotations.*;
|
||||
|
|
@ -13,9 +12,13 @@ import edu.stanford.nlp.semgraph.SemanticGraph;
|
|||
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations;
|
||||
import edu.stanford.nlp.semgraph.SemanticGraphEdge;
|
||||
import edu.stanford.nlp.util.CoreMap;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.base.Joiner;
|
||||
|
||||
import fig.basic.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
|
|
@ -113,6 +116,7 @@ public class CoreNLPAnalyzer extends LanguageAnalyzer {
|
|||
// Fills in a stanford dependency graph for constructing a feature
|
||||
for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
|
||||
SemanticGraph ccDeps = sentence.get(SemanticGraphCoreAnnotations.CollapsedCCProcessedDependenciesAnnotation.class);
|
||||
if (ccDeps == null) continue;
|
||||
int sentenceBegin = sentence.get(CoreAnnotations.TokenBeginAnnotation.class);
|
||||
|
||||
// Iterate over all tokens and their dependencies
|
||||
|
|
@ -133,4 +137,27 @@ public class CoreNLPAnalyzer extends LanguageAnalyzer {
|
|||
}
|
||||
return languageInfo;
|
||||
}
|
||||
|
||||
// Test on example sentence.
|
||||
public static void main(String[] args) {
|
||||
CoreNLPAnalyzer analyzer = new CoreNLPAnalyzer();
|
||||
while (true) {
|
||||
try {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
|
||||
System.out.println("Enter some text:");
|
||||
String text = reader.readLine();
|
||||
LanguageInfo langInfo = analyzer.analyze(text);
|
||||
LogInfo.begin_track("Analyzing \"%s\"", text);
|
||||
LogInfo.logs("tokens: %s", langInfo.tokens);
|
||||
LogInfo.logs("lemmaTokens: %s", langInfo.lemmaTokens);
|
||||
LogInfo.logs("posTags: %s", langInfo.posTags);
|
||||
LogInfo.logs("nerTags: %s", langInfo.nerTags);
|
||||
LogInfo.logs("nerValues: %s", langInfo.nerValues);
|
||||
LogInfo.logs("dependencyChildren: %s", langInfo.dependencyChildren);
|
||||
LogInfo.end_track();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
<project>
|
||||
<target name="compile">
|
||||
<property name="module_name" value="corenlp"/>
|
||||
<property name="root" location="../../../../../../"/>
|
||||
<property name="classes" location="${root}/classes/${module_name}"/>
|
||||
<mkdir dir="${classes}"/>
|
||||
<mkdir dir="${root}/libsempre"/>
|
||||
<path id="lib.path"><fileset dir="${root}/lib"/><fileset dir="${root}/libsempre"/></path>
|
||||
<javac srcdir="${root}/src" destdir="${classes}" classpathref="lib.path" debug="true" includeantruntime="false">
|
||||
<include name="edu/stanford/nlp/sempre/corenlp/"/>
|
||||
</javac>
|
||||
<jar destfile="${root}/libsempre/sempre-${module_name}.jar" basedir="${classes}"/>
|
||||
</target>
|
||||
</project>
|
||||
|
|
@ -87,7 +87,7 @@ public class Free917Converter implements Runnable {
|
|||
questionWriter.println("(lambda $0 /common/topic (exists $1 (exists $2 (/film/film/estimated_budget&/measurement_unit/dated_money_value@valid_date@amount@currency:t /en/transformers:/film/film $-1 $1 $0 $2))))");
|
||||
else if (line.equals("(lambda $0 /location/location (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/library/public_library/address&/location/mailing_address@street_address@street_address_2@citytown@postal_code@state_province_region@country:t /m/02ncllz:/library/public_library $1 $2 $0 $3 $4 $5)))))))"))
|
||||
questionWriter.println("(lambda $0 /location/location (exists $1 (exists $2 (exists $3 (exists $4 (exists $5 (/library/public_library/address&/location/mailing_address@street_address@street_address_2@citytown@postal_code@state_province_region@country:t /m/02ncllz:/library/public_library $-1 $1 $2 $0 $3 $4 $5)))))))");
|
||||
else if (line.equals("who won ali–frazier ii"))
|
||||
else if (line.matches("who won ali.*frazier ii"))
|
||||
questionWriter.println("who won muhammad ali vs. joe frazier ii");
|
||||
else if (line.equals("(lambda $0 /people/person (exists $1 (exists $2 (exists $3 (/base/boxing/match_boxer_relationship@match@boxer@winner_won@points:t $1 $0 $2 $3)))))")) {
|
||||
if (i++ == 0)
|
||||
|
|
@ -341,7 +341,7 @@ public class Free917Converter implements Runnable {
|
|||
npWriter.println("film domain :- NP : /m/010s:/freebase/domain_profile");
|
||||
else if (line.equals("newscaster :- NP : /en/newscaster:/tv/non_character_role"))
|
||||
npWriter.println("newscaster :- NP : /en/news_presenter:/tv/non_character_role");
|
||||
else if (line.equals("ali<EFBFBD>frazier ii :- NP : /en/ali-frazier_ii:/base/boxing/boxing_match")) {
|
||||
else if (line.matches("ali.*frazier ii :- NP : /en/ali-frazier_ii:/base/boxing/boxing_match")) {
|
||||
} else
|
||||
npWriter.println(line);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
package edu.stanford.nlp.sempre.freebase;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
import fig.basic.*;
|
||||
import fig.exec.Execution;
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Converts Luke Zettlemoyer's lambda calculus data format into our example files.
|
||||
|
|
@ -15,25 +18,25 @@ import edu.stanford.nlp.sempre.*;
|
|||
public class LambdaCalculusConverter implements Runnable {
|
||||
public static class Options {
|
||||
@Option(gloss = "Input path (lambda calculus)")
|
||||
public String inPath = "freebase/data/geo/geosents600-typed.ccg.dev";
|
||||
public String inPath = "overnight/geo/geosents280-typed.ccg.test.new";
|
||||
@Option(gloss = "Specification of translations")
|
||||
public String specPath = "freebase/data/geo/geo.spec";
|
||||
public String specPath = "overnight/geo/geo.spec";
|
||||
@Option(gloss = "Specification of variable names")
|
||||
public String varPath = "freebase/data/geo/geo.vars";
|
||||
public String varPath = "overnight/geo/geo.vars";
|
||||
@Option(gloss = "Specification of primitive types")
|
||||
public String primPath = "freebase/data/geo/geo.primitives";
|
||||
public String primPath = "overnight/geo/geo.primitives";
|
||||
@Option(gloss = "Specification of formula replacements")
|
||||
public String replacePath = "freebase/data/geo/geo.replace";
|
||||
public String replacePath = "overnight/geo/geo.replace";
|
||||
@Option(gloss = "Specification of manual conversions")
|
||||
public String manualConversionsPath = "freebase/data/geo/geo.manual_conversions";
|
||||
@Option(gloss = "Input path (examples)")
|
||||
public String outPath = "freebase/data/geo/geo.out.json";
|
||||
public String manualConversionsPath = "overnight/geo/geo.manual_conversions";
|
||||
@Option(gloss = "Output path (examples)")
|
||||
public String outPath = "overnight/geo/geo.out.json";
|
||||
@Option(gloss = "Specific example to parse and run")
|
||||
public int runInd = -1;
|
||||
public int runInd = -1;
|
||||
@Option(gloss = "Output path for lexicon grammar")
|
||||
public String lexiconPath = "freebase/data/geo/geo.out.grammar";
|
||||
public String lexiconPath = "overnight/geo/geo.out.grammar";
|
||||
@Option(gloss = "Verbose output (for debugging)")
|
||||
public boolean verbose = false; // TODO Currently unused
|
||||
public boolean verbose = false; // TODO Currently unused
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
|
|
@ -72,11 +75,13 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
}
|
||||
|
||||
public void readPrereqs() {
|
||||
LogInfo.begin_track("Reading prereq");
|
||||
readSpec();
|
||||
readPrimitives();
|
||||
readVars();
|
||||
readStringMap(opts.replacePath, replaceMap);
|
||||
readStringMap(opts.manualConversionsPath, manualConversionsMap);
|
||||
LogInfo.end_track();
|
||||
}
|
||||
|
||||
void readSpec() {
|
||||
|
|
@ -248,7 +253,7 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
if (primitive == null)
|
||||
return null;
|
||||
// if (primitive.equals("string"))
|
||||
// parts[0] = capitalizeWords(parts[0]);
|
||||
// parts[0] = capitalizeWords(parts[0]);
|
||||
// return Formula.fromString(String.format("(%s %s)", primitive, parts[0]));
|
||||
if (primitive.equals("string"))
|
||||
return Formula.fromString(String.format("(!fb:type.object.name fb:en.%s)", parts[0]));
|
||||
|
|
@ -257,41 +262,41 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
}
|
||||
|
||||
Formula handleExists(LispTree tree, String headVar, List<String> existsVars) {
|
||||
// (exists $1 (and (river:t $1) (loc:t $1 $0)))
|
||||
String eVar = tree.child(1).value;
|
||||
existsVars.add(eVar);
|
||||
// FIXME Currently assuming format where exists contains and statement with
|
||||
// n clauses where first n - 1 clauses describe exists var and last clause
|
||||
// relates exists var to the head var
|
||||
List<LispTree> andTreeList = tree.child(2).children;
|
||||
// FIXME Don't assume and tree
|
||||
LispTree newAndTree = LispTree.proto.newList();
|
||||
// (exists $1 (and (river:t $1) (loc:t $1 $0)))
|
||||
String eVar = tree.child(1).value;
|
||||
existsVars.add(eVar);
|
||||
// FIXME Currently assuming format where exists contains and statement with
|
||||
// n clauses where first n - 1 clauses describe exists var and last clause
|
||||
// relates exists var to the head var
|
||||
List<LispTree> andTreeList = tree.child(2).children;
|
||||
// FIXME Don't assume and tree
|
||||
LispTree newAndTree = LispTree.proto.newList();
|
||||
|
||||
int predTreeInd = -1;
|
||||
int numPredTrees = 0;
|
||||
for (int k = 0; k < andTreeList.size(); k++) {
|
||||
String childStr = andTreeList.get(k).toString();
|
||||
if (childStr.contains(headVar)) {
|
||||
predTreeInd = k;
|
||||
numPredTrees++;
|
||||
}
|
||||
int predTreeInd = -1;
|
||||
int numPredTrees = 0;
|
||||
for (int k = 0; k < andTreeList.size(); k++) {
|
||||
String childStr = andTreeList.get(k).toString();
|
||||
if (childStr.contains(headVar)) {
|
||||
predTreeInd = k;
|
||||
numPredTrees++;
|
||||
}
|
||||
}
|
||||
|
||||
String func = tree.child(2).child(0).value;
|
||||
if (numPredTrees == 0 || func.equals("exists")) {
|
||||
return toLambda(eVar, toFormula(tree.child(2), eVar, existsVars));
|
||||
}
|
||||
String func = tree.child(2).child(0).value;
|
||||
if (numPredTrees == 0 || func.equals("exists")) {
|
||||
return toLambda(eVar, toFormula(tree.child(2), eVar, existsVars));
|
||||
}
|
||||
|
||||
newAndTree.children = new ArrayList<LispTree>(andTreeList.subList(0, andTreeList.size()));
|
||||
LispTree predTree = newAndTree.children.remove(predTreeInd);
|
||||
if (newAndTree.children.size() == 2)
|
||||
newAndTree = newAndTree.child(1);
|
||||
newAndTree.children = new ArrayList<LispTree>(andTreeList.subList(0, andTreeList.size()));
|
||||
LispTree predTree = newAndTree.children.remove(predTreeInd);
|
||||
if (newAndTree.children.size() == 2)
|
||||
newAndTree = newAndTree.child(1);
|
||||
|
||||
LispTree newPredTree = LispTree.proto.parseFromString(predTree.toString().replace(eVar, newAndTree.toString()));
|
||||
LispTree newPredTree = LispTree.proto.parseFromString(predTree.toString().replace(eVar, newAndTree.toString()));
|
||||
|
||||
Formula form = toFormula(newPredTree, eVar, existsVars);
|
||||
existsVars.remove(eVar);
|
||||
return form;
|
||||
Formula form = toFormula(newPredTree, eVar, existsVars);
|
||||
existsVars.remove(eVar);
|
||||
return form;
|
||||
}
|
||||
|
||||
// hit: ignore these clauses in (and ...) constructions
|
||||
|
|
@ -323,7 +328,7 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
// (sum $0 (and (state:t $0) (next_to:t $0 texas:s)) (population:i $0))
|
||||
String numFunc = tree.child(3).child(0).value;
|
||||
return new AggregateFormula(AggregateFormula.Mode.sum,
|
||||
toJoin(numFunc, toFormula(tree.child(2), tree.child(1).value, existsVars)));
|
||||
toJoin(numFunc, toFormula(tree.child(2), tree.child(1).value, existsVars)));
|
||||
} else if (func.equals("argmax") || func.equals("argmin")) {
|
||||
// (argmax $0 (state:t $0) (density:i $0))
|
||||
// (argmin $1 (river:t $1) (len:i $1))
|
||||
|
|
@ -379,7 +384,7 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
else // secondIsHead
|
||||
return toJoin("!" + func, form1);
|
||||
// else
|
||||
// return toJoin(func, form1);
|
||||
// return toJoin(func, form1);
|
||||
} else {
|
||||
throw new RuntimeException("Bad arity: " + tree);
|
||||
}
|
||||
|
|
@ -458,8 +463,10 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
ArrayList<String> existsVars = new ArrayList<String>();
|
||||
|
||||
if (manualConversionsMap.containsKey(line)) {
|
||||
LogInfo.logs("MANUAL CONVERSION=%s", line);
|
||||
ex.setTargetFormula(Formula.fromString(manualConversionsMap.get(line)));
|
||||
} else {
|
||||
LogInfo.log("AUTOMATIC CONVERSION");
|
||||
ex.setTargetFormula(toFormula(tree, null, existsVars));
|
||||
}
|
||||
LogInfo.logs(color.colorize("OUT [%d]: %s", "yellow"), newId, ex.createExample().targetFormula.toLispTree());
|
||||
|
|
@ -476,8 +483,9 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
void executeExamples() {
|
||||
int runInd = opts.runInd;
|
||||
// FIXME Should be passed in
|
||||
SparqlExecutor.opts.endpointUrl = "http://localhost:3093/sparql";
|
||||
SparqlExecutor.opts.endpointUrl = "http://localhost:3094/sparql";
|
||||
SparqlExecutor.opts.cachePath = "SparqlExecutor.cache";
|
||||
SparqlExecutor.opts.lambdaAllowDiagonals = false; //jonathan
|
||||
SparqlExecutor executor = new SparqlExecutor();
|
||||
|
||||
int exInd = 1;
|
||||
|
|
@ -506,12 +514,12 @@ public class LambdaCalculusConverter implements Runnable {
|
|||
|
||||
void writeExamples() {
|
||||
// Sort validExampleIds by the length of the utterance of the example
|
||||
Collections.sort(validExampleIds, new Comparator<Integer>() {
|
||||
public int compare(Integer left, Integer right) {
|
||||
return Integer.compare(validExampleLengths.get(validExampleIds.indexOf(left)),
|
||||
validExampleLengths.get(validExampleIds.indexOf(right)));
|
||||
}
|
||||
});
|
||||
// Collections.sort(validExampleIds, new Comparator<Integer>() {
|
||||
// public int compare(Integer left, Integer right) {
|
||||
// return Integer.compare(validExampleLengths.get(validExampleIds.indexOf(left)),
|
||||
// validExampleLengths.get(validExampleIds.indexOf(right)));
|
||||
// }
|
||||
// });
|
||||
|
||||
PrintWriter out = IOUtils.openOutHard(opts.outPath);
|
||||
out.println("["); // Print out as a list
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
<project>
|
||||
<target name="compile">
|
||||
<property name="module_name" value="freebase"/>
|
||||
<property name="root" location="../../../../../../"/>
|
||||
<property name="classes" location="${root}/classes/${module_name}"/>
|
||||
<mkdir dir="${classes}"/>
|
||||
<mkdir dir="${root}/libsempre"/>
|
||||
<path id="lib.path"><fileset dir="${root}/lib"/><fileset dir="${root}/libsempre"/></path>
|
||||
<javac srcdir="${root}/src" destdir="${classes}" classpathref="lib.path" debug="true" includeantruntime="false">
|
||||
<include name="edu/stanford/nlp/sempre/freebase/"/>
|
||||
</javac>
|
||||
<jar destfile="${root}/libsempre/sempre-${module_name}.jar" basedir="${classes}"/>
|
||||
</target>
|
||||
</project>
|
||||
|
|
@ -69,8 +69,10 @@ public class SparqlExecutorTest {
|
|||
SparqlExecutor executor = new SparqlExecutor();
|
||||
|
||||
public SparqlExecutorTest() {
|
||||
SparqlExecutor.opts.endpointUrl = System.getProperty("sparqlserver");
|
||||
// Hard-coding not ideal.
|
||||
SparqlExecutor.opts.endpointUrl = "http://localhost:3093/sparql";
|
||||
if (SparqlExecutor.opts.endpointUrl == null)
|
||||
SparqlExecutor.opts.endpointUrl = "http://freebase.cloudapp.net:3093/sparql";
|
||||
SparqlExecutor.opts.verbose = 3;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,18 @@
|
|||
package edu.stanford.nlp.sempre.freebase.utils;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.BiMap;
|
||||
import com.google.common.collect.HashBiMap;
|
||||
import edu.stanford.nlp.io.IOUtils;
|
||||
import edu.stanford.nlp.objectbank.ObjectBank;
|
||||
import edu.stanford.nlp.sempre.DateValue;
|
||||
import edu.stanford.nlp.sempre.DescriptionValue;
|
||||
import edu.stanford.nlp.sempre.Json;
|
||||
import edu.stanford.nlp.sempre.NameValue;
|
||||
import edu.stanford.nlp.stats.ClassicCounter;
|
||||
import edu.stanford.nlp.stats.Counter;
|
||||
import edu.stanford.nlp.util.StringUtils;
|
||||
import fig.basic.LispTree;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
|
|
@ -202,4 +208,54 @@ public final class FileUtils {
|
|||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
//input: tab separated file with |utterance| |gold| |predicted|
|
||||
//output: prediction file for codalab
|
||||
public static void generatePredictionFile(String inFile, String outFile) throws IOException {
|
||||
|
||||
PrintWriter writer = IOUtils.getPrintWriter(outFile);
|
||||
for (String line : IOUtils.readLines(inFile)) {
|
||||
String[] tokens = line.split("\\t");
|
||||
if (tokens.length == 0)
|
||||
continue;
|
||||
if (tokens.length < 2)
|
||||
throw new RuntimeException("Illegal line: " + line);
|
||||
String utterance = tokens[0];
|
||||
// get gold
|
||||
List<String> goldDescriptions = new ArrayList<>();
|
||||
LispTree goldTree = LispTree.proto.parseFromString(tokens[1]);
|
||||
for (int i = 1; i < goldTree.children.size(); ++i) {
|
||||
DescriptionValue dValue = (DescriptionValue) DescriptionValue.fromString(goldTree.child(i).toString());
|
||||
goldDescriptions.add(dValue.value);
|
||||
}
|
||||
// get predicted
|
||||
List<String> predictedDescriptions = new ArrayList<>();
|
||||
if (tokens.length > 2) {
|
||||
LispTree predictedTree = LispTree.proto.parseFromString(tokens[2]);
|
||||
for (int i = 1; i < predictedTree.children.size(); ++i) {
|
||||
if (predictedTree.child(i).child(0).value.equals("name")) {
|
||||
NameValue nValue = (NameValue) NameValue.fromString(predictedTree.child(i).toString());
|
||||
predictedDescriptions.add(nValue.description);
|
||||
} else if (predictedTree.child(i).child(0).value.equals("date")) {
|
||||
DateValue dateValue = (DateValue) DateValue.fromString(predictedTree.child(i).toString());
|
||||
predictedDescriptions.add(dateValue.toString());
|
||||
} else
|
||||
throw new RuntimeException("Can not support this value: " + line);
|
||||
}
|
||||
}
|
||||
writer.println(Joiner.on('\t').join(utterance,
|
||||
Json.writeValueAsStringHard(goldDescriptions),
|
||||
Json.writeValueAsStringHard(predictedDescriptions)));
|
||||
}
|
||||
writer.close();
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
generatePredictionFile(args[0], args[1]);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,146 @@
|
|||
package edu.stanford.nlp.sempre.overnight;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import edu.stanford.nlp.io.IOUtils;
|
||||
import edu.stanford.nlp.stats.ClassicCounter;
|
||||
import edu.stanford.nlp.stats.Counter;
|
||||
import edu.stanford.nlp.stats.Counters;
|
||||
import fig.basic.LispTree;
|
||||
import fig.basic.MapUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Word-aligns original utterances with their paraphrases
|
||||
* Created by joberant on 2/20/15.
|
||||
*/
|
||||
public class Aligner {
|
||||
|
||||
private Map<String, Counter<String>> model = new HashMap<>();
|
||||
|
||||
public double getCondProb(String target, String source) {
|
||||
if (model.containsKey(source)) {
|
||||
if (model.get(source).containsKey(target))
|
||||
return model.get(source).getCount(target);
|
||||
}
|
||||
return 0d;
|
||||
}
|
||||
|
||||
public int size() { return model.size(); }
|
||||
|
||||
//takes an example file and creates a model
|
||||
public void heuristicAlign(String exampleFile, int threshold) {
|
||||
|
||||
model.clear();
|
||||
Iterator<LispTree> iter = LispTree.proto.parseFromFile(exampleFile);
|
||||
|
||||
while (iter.hasNext()) {
|
||||
LispTree tree = iter.next();
|
||||
LispTree utteranceTree = tree.child(1);
|
||||
LispTree originalTree = tree.child(2);
|
||||
String utterance = preprocessUtterance(utteranceTree.child(1).value);
|
||||
String original = preprocessUtterance(originalTree.child(1).value);
|
||||
String[] utteranceTokens = utterance.split("\\s+");
|
||||
String[] originalTokens = original.split("\\s+");
|
||||
|
||||
align(utteranceTokens, originalTokens);
|
||||
}
|
||||
normalize(threshold);
|
||||
}
|
||||
|
||||
public void saveModel(String out) throws IOException {
|
||||
PrintWriter writer = IOUtils.getPrintWriter(out);
|
||||
for (String source: model.keySet()) {
|
||||
Counter<String> counts = model.get(source);
|
||||
for (String target: counts.keySet()) {
|
||||
writer.println(Joiner.on('\t').join(source, target, counts.getCount(target)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//normalize all the counts to get conditional probabilities
|
||||
private void normalize(int threshold) {
|
||||
for (String source: model.keySet()) {
|
||||
Counter<String> counts = model.get(source);
|
||||
Counters.removeKeys(counts, Counters.keysBelow(counts, threshold));
|
||||
Counters.normalize(counts);
|
||||
}
|
||||
}
|
||||
|
||||
//count every co-occurrence
|
||||
private void align(String[] utteranceTokens, String[] originalTokens) {
|
||||
for (String utteranceToken: utteranceTokens) {
|
||||
for (String originalToken: originalTokens) {
|
||||
MapUtils.putIfAbsent(model, utteranceToken.toLowerCase(), new ClassicCounter<>());
|
||||
MapUtils.putIfAbsent(model, originalToken.toLowerCase(), new ClassicCounter<>());
|
||||
model.get(utteranceToken.toLowerCase()).incrementCount(originalToken.toLowerCase());
|
||||
model.get(originalToken.toLowerCase()).incrementCount(utteranceToken.toLowerCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//remove '?' and '.'
|
||||
public String preprocessUtterance(String utterance) {
|
||||
if (utterance.endsWith("?"))
|
||||
return utterance.substring(0, utterance.length() - 1);
|
||||
if (utterance.endsWith("."))
|
||||
return utterance.substring(0, utterance.length() - 1);
|
||||
return utterance;
|
||||
}
|
||||
|
||||
//read from serialized file
|
||||
public static Aligner read(String path) {
|
||||
Aligner res = new Aligner();
|
||||
for (String line: edu.stanford.nlp.io.IOUtils.readLines(path)) {
|
||||
String[] tokens = line.split("\t");
|
||||
MapUtils.putIfAbsent(res.model, tokens[0], new ClassicCounter<>());
|
||||
res.model.get(tokens[0]).incrementCount(tokens[1], Double.parseDouble(tokens[2]));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private void berkeleyAlign(String file, int threshold) {
|
||||
for (String line: IOUtils.readLines(file)) {
|
||||
String[] tokens = line.split("\t");
|
||||
String[] sourceTokens = tokens[0].split("\\s+");
|
||||
String[] targetTokens = tokens[1].split("\\s+");
|
||||
String[] alignmentTokens = tokens[2].split("\\s+");
|
||||
for (String alignmentToken: alignmentTokens) {
|
||||
String[] alignment = alignmentToken.split("-");
|
||||
Integer source = Integer.parseInt(alignment[0]);
|
||||
Integer target = Integer.parseInt(alignment[1]);
|
||||
MapUtils.putIfAbsent(model, sourceTokens[source], new ClassicCounter<>());
|
||||
MapUtils.putIfAbsent(model, targetTokens[target], new ClassicCounter<>());
|
||||
model.get(sourceTokens[source]).incrementCount(targetTokens[target]);
|
||||
model.get(targetTokens[target]).incrementCount(sourceTokens[source]);
|
||||
}
|
||||
}
|
||||
normalize(threshold);
|
||||
}
|
||||
|
||||
//args[0] - example file with utterance and original
|
||||
//args[1] - output file
|
||||
//args[2] - heuristic or berkeley
|
||||
//args[3] - threshold
|
||||
public static void main(String[] args) {
|
||||
Aligner aligner = new Aligner();
|
||||
int threshold = Integer.parseInt(args[3]);
|
||||
if (args[2].equals("heuristic"))
|
||||
aligner.heuristicAlign(args[0], threshold);
|
||||
else if (args[2].equals("berkeley"))
|
||||
aligner.berkeleyAlign(args[0], threshold);
|
||||
else throw new RuntimeException("bad alignment mode: " + args[2]);
|
||||
try {
|
||||
aligner.saveModel(args[1]);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
package edu.stanford.nlp.sempre.overnight;
|
||||
|
||||
import edu.stanford.nlp.io.IOUtils;
|
||||
import edu.stanford.nlp.sempre.StringValue;
|
||||
import fig.basic.LispTree;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by joberant on 2/24/15.
|
||||
* This converts
|
||||
* (targetValue (list (name fb:en.place.walton_county)))
|
||||
* to
|
||||
* (targetValue (string [(name fb:en.place.walton_county)]))
|
||||
* This has nothing to do with 'paraphrase', it's just here for no reason
|
||||
*/
|
||||
public final class ConvertTargetValueFromListToString {
|
||||
private ConvertTargetValueFromListToString() { }
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
try {
|
||||
PrintWriter writer = IOUtils.getPrintWriter(args[1]);
|
||||
Iterator<LispTree> trees = LispTree.proto.parseFromFile(args[0]);
|
||||
while (trees.hasNext()) {
|
||||
LispTree tree = trees.next();
|
||||
|
||||
LispTree outTree = LispTree.proto.newList();
|
||||
outTree.addChild("example");
|
||||
outTree.addChild(tree.child(1));
|
||||
|
||||
List<String> output = new ArrayList<>();
|
||||
LispTree targetValue = tree.child(3);
|
||||
if (!targetValue.child(0).value.equals("targetValue"))
|
||||
throw new RuntimeException("Expected a target value as second child: " + targetValue);
|
||||
LispTree list = targetValue.child(1);
|
||||
if (!list.child(0).value.equals("list"))
|
||||
throw new RuntimeException("Expected a list as first child: " + list);
|
||||
for (int i = 1; i < list.children.size(); ++i)
|
||||
output.add(list.child(i).toString());
|
||||
StringValue newTargetValue = new StringValue(output.toString());
|
||||
LispTree newTargetValueTree = LispTree.proto.newList();
|
||||
newTargetValueTree.addChild("targetValue");
|
||||
newTargetValueTree.addChild(newTargetValue.toLispTree());
|
||||
outTree.addChild(newTargetValueTree);
|
||||
outTree.print(120, 120, writer);
|
||||
writer.println();
|
||||
}
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
package edu.stanford.nlp.sempre.overnight;
|
||||
|
||||
import edu.stanford.nlp.io.IOUtils;
|
||||
import fig.basic.LispTree;
|
||||
import fig.basic.LogInfo;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* Created by joberant on 2/22/15.
|
||||
* Takes a file with lisp trees of original and canonical utterances
|
||||
* and creates the input for the berkley aligner
|
||||
*/
|
||||
public final class CreateBerkeleyAlignerInputFromLispTree {
|
||||
private CreateBerkeleyAlignerInputFromLispTree() { }
|
||||
|
||||
//args[0]: lisp tree file
|
||||
//args[1] output directory
|
||||
public static void main(String[] args) {
|
||||
|
||||
Iterator<LispTree> trees = LispTree.proto.parseFromFile(args[0]);
|
||||
try {
|
||||
PrintWriter writerOriginal = IOUtils.getPrintWriter(args[1] + ".e");
|
||||
PrintWriter writerUtterance = IOUtils.getPrintWriter(args[1] + ".f");
|
||||
LogInfo.logs("output directory=%s", args[1]);
|
||||
|
||||
int i = 0;
|
||||
while (trees.hasNext()) {
|
||||
i++;
|
||||
LispTree tree = trees.next();
|
||||
LispTree utteranceTree = tree.child(1);
|
||||
LispTree originalTree = tree.child(2);
|
||||
if (!utteranceTree.child(0).value.equals("utterance"))
|
||||
throw new RuntimeException("First child is not an utterance " + utteranceTree);
|
||||
if (!originalTree.child(0).value.equals("original"))
|
||||
throw new RuntimeException("second child is not the original " + originalTree);
|
||||
String uttearnce = utteranceTree.child(1).value;
|
||||
if (uttearnce.endsWith("?") || uttearnce.endsWith("."))
|
||||
uttearnce = uttearnce.substring(0, uttearnce.length() - 1);
|
||||
String original = originalTree.child(1).value;
|
||||
writerOriginal.println(original);
|
||||
writerUtterance.println(uttearnce);
|
||||
}
|
||||
LogInfo.logs("Numebr of trees=%s", i);
|
||||
writerOriginal.close();
|
||||
writerUtterance.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package edu.stanford.nlp.sempre.overnight;
|
||||
|
||||
import fig.basic.Option;
|
||||
import fig.exec.Execution;
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
|
||||
/**
|
||||
* Created by joberant on 1/27/15.
|
||||
* Generating canonical utterances from grammar with various depths
|
||||
*/
|
||||
public class GenerationMain implements Runnable {
|
||||
@Option
|
||||
public boolean interactive = false;
|
||||
@Option
|
||||
public boolean varyMaxDepth = false;
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
Builder builder = new Builder();
|
||||
builder.build();
|
||||
|
||||
Dataset dataset = new Dataset();
|
||||
dataset.read();
|
||||
|
||||
int currDepth = varyMaxDepth ? 1 : FloatingParser.opts.maxDepth;
|
||||
int maxDepth = FloatingParser.opts.maxDepth;
|
||||
|
||||
for (; currDepth < maxDepth + 1; currDepth++) {
|
||||
FloatingParser.opts.maxDepth = currDepth;
|
||||
//LogInfo.logs("Curr depth=%s", currDepth);
|
||||
//LogInfo.logs("file = %s", FloatingParser.opts.predictedUtterancesFile);
|
||||
//PrintWriter writer = IOUtils.openOutAppendEasy(Execution.getFile(FloatingParser.opts.predictedUtterancesFile));
|
||||
//writer.println(String.format("Depth=%s", currDepth));
|
||||
//writer.println(String.format("--------", currDepth));
|
||||
//writer.close();
|
||||
Learner learner = new Learner(builder.parser, builder.params, dataset);
|
||||
learner.learn();
|
||||
}
|
||||
|
||||
|
||||
if (interactive) {
|
||||
Master master = new Master(builder);
|
||||
master.runInteractivePrompt();
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
Execution.run(args, "GenerationMain", new GenerationMain(), Master.getOptionsParser());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
package edu.stanford.nlp.sempre.overnight;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
import fig.basic.Option;
|
||||
import fig.basic.MapUtils;
|
||||
|
||||
/**
|
||||
* Hard-coded hacks for pruning derivations in floating parser for overnight domains.
|
||||
*
|
||||
*/
|
||||
|
||||
public class OvernightDerivationPruningComputer extends DerivationPruningComputer {
|
||||
public static class Options {
|
||||
@Option (gloss = "Whether filter derivations using hard constraints")
|
||||
public boolean applyHardConstraints = false;
|
||||
}
|
||||
public static Options opts = new Options();
|
||||
|
||||
|
||||
public OvernightDerivationPruningComputer(DerivationPruner pruner) {
|
||||
super(pruner);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPruned(Derivation deriv) {
|
||||
if (opts.applyHardConstraints && violateHardConstraints(deriv)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check a few hard constraints on each derivation
|
||||
private static boolean violateHardConstraints(Derivation deriv) {
|
||||
if (deriv.value != null) {
|
||||
if (deriv.value instanceof ErrorValue) return true;
|
||||
if (deriv.value instanceof StringValue) { //empty denotation
|
||||
if (((StringValue) deriv.value).value.equals("[]")) return true;
|
||||
}
|
||||
if (deriv.value instanceof ListValue) {
|
||||
List<Value> values = ((ListValue) deriv.value).values;
|
||||
// empty lists
|
||||
if (values.size() == 0) return true;
|
||||
// NaN
|
||||
if (values.size() == 1 && values.get(0) instanceof NumberValue) {
|
||||
if (Double.isNaN(((NumberValue) values.get(0)).value)) return true;
|
||||
}
|
||||
// If we are supposed to get a number but we get a string (some sparql weirdness)
|
||||
if (deriv.type.equals(SemType.numberType) &&
|
||||
values.size() == 1 &&
|
||||
!(values.get(0) instanceof NumberValue)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,609 @@
|
|||
package edu.stanford.nlp.sempre.overnight;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.Sets;
|
||||
import edu.stanford.nlp.io.IOUtils;
|
||||
import fig.basic.BipartiteMatcher;
|
||||
import fig.basic.LogInfo;
|
||||
import fig.basic.MapUtils;
|
||||
import fig.basic.Option;
|
||||
import edu.stanford.nlp.sempre.*;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Define features on the input utterance and a partial canonical utterance.
|
||||
*
|
||||
* Feature computation recipe:
|
||||
* - For both the input and (partial) canonical utterance, extract a list of tokens
|
||||
* (perhaps with POS tags).
|
||||
* - Given a list of tokens, extract a set of items, where an item is a (tag,
|
||||
* data) pair, where the tag specifies the "type" of the data, and is used
|
||||
* to determine features. Example: ("bigram", "not contains"), ("unigram",
|
||||
* "not"), ("unigram-RB", "not")
|
||||
* - Given the input and canonical items, define recall features (how much of
|
||||
* the input items is the canononical covering).
|
||||
* This recipe allows us to decouple the extraction of items on one utterance
|
||||
* from the computation of actual precision/recall features.
|
||||
*
|
||||
* @author Percy Liang
|
||||
* @author Yushi Wang
|
||||
*/
|
||||
public class OvernightFeatureComputer implements FeatureComputer {
|
||||
public static class Options {
|
||||
@Option(gloss = "Set of paraphrasing feature domains to include")
|
||||
public Set<String> featureDomains = new HashSet<>();
|
||||
|
||||
@Option(gloss = "Whether or not to count intermediate categories for size feature")
|
||||
public boolean countIntermediate = true;
|
||||
|
||||
@Option(gloss = "Whether or not to do match/ppdb analysis")
|
||||
public boolean itemAnalysis = true;
|
||||
|
||||
@Option(gloss = "Whether or not to learn paraphrases")
|
||||
public boolean learnParaphrase = true;
|
||||
|
||||
@Option(gloss = "Verbose flag")
|
||||
public int verbose = 0;
|
||||
|
||||
@Option(gloss = "Path to alignment file")
|
||||
public String wordAlignmentPath;
|
||||
@Option(gloss = "Path to phrase alignment file")
|
||||
public String phraseAlignmentPath;
|
||||
@Option(gloss = "Threshold for phrase table co-occurrence")
|
||||
public int phraseTableThreshold = 3;
|
||||
}
|
||||
|
||||
public static Options opts = new Options();
|
||||
|
||||
private static Aligner aligner;
|
||||
private static Map<String, Map<String, Double>> phraseTable;
|
||||
public final SimpleLexicon simpleLexicon = SimpleLexicon.getSingleton();
|
||||
|
||||
@Override public void extractLocal(Example ex, Derivation deriv) {
|
||||
if (deriv.rule.rhs == null) return;
|
||||
|
||||
// Optimization: feature vector same as child, so don't do anything.
|
||||
if (deriv.rule.isCatUnary()) {
|
||||
if (deriv.isRootCat()) {
|
||||
extractValueInFormulaFeature(deriv);
|
||||
extractRootFeatures(ex, deriv);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Important! We want to define the global feature vector for this
|
||||
// derivation, but we can only specify the local feature vector. So to
|
||||
// make things cancel out, we subtract out the unwanted feature vectors of
|
||||
// descendents.
|
||||
subtractDescendentsFeatures(deriv, deriv);
|
||||
|
||||
deriv.addFeature("paraphrase", "size", derivationSize(deriv));
|
||||
extractRootFeatures(ex, deriv);
|
||||
extractLexicalFeatures(ex, deriv);
|
||||
extractPhraseAlignmentFeatures(ex, deriv);
|
||||
extractLogicalFormFeatures(ex, deriv);
|
||||
|
||||
if (!opts.itemAnalysis) return;
|
||||
|
||||
List<Item> inputItems = computeInputItems(ex);
|
||||
List<Item> candidateItems = computeCandidateItems(ex, deriv);
|
||||
|
||||
for (Item input : inputItems) {
|
||||
double match = 0;
|
||||
double ppdb = 0;
|
||||
double skipBigram = 0;
|
||||
double skipPpdb = 0;
|
||||
for (Item candidate : candidateItems) {
|
||||
if (!input.tag.equals(candidate.tag)) continue;
|
||||
if (input.tag.equals("skip-bigram")) {
|
||||
skipBigram = Math.max(skipBigram, computeMatch(input.data, candidate.data));
|
||||
skipPpdb = Math.max(skipPpdb, computeParaphrase(input.data, candidate.data));
|
||||
} else {
|
||||
|
||||
match = Math.max(match, computeMatch(input.data, candidate.data));
|
||||
ppdb = Math.max(ppdb, computeParaphrase(input.data, candidate.data));
|
||||
}
|
||||
}
|
||||
if (match > 0 && opts.featureDomains.contains("match")) deriv.addFeature("paraphrase", "match");
|
||||
if (ppdb > 0 && opts.featureDomains.contains("ppdb")) deriv.addFeature("paraphrase", "ppdb");
|
||||
if (skipBigram > 0 && opts.featureDomains.contains("skip-bigram")) deriv.addFeature("paraphrase", "skip-bigram");
|
||||
if (skipPpdb > 0 && opts.featureDomains.contains("skip-ppdb")) deriv.addFeature("paraphrase", "skip-ppdb");
|
||||
}
|
||||
|
||||
HashMap<String, Double> features = new LinkedHashMap<>();
|
||||
deriv.incrementAllFeatureVector(+1, features);
|
||||
if (opts.verbose >= 1) {
|
||||
LogInfo.logs("category %s, %s %s", deriv.cat, inputItems, candidateItems);
|
||||
FeatureVector.logFeatures(features);
|
||||
}
|
||||
}
|
||||
|
||||
private void extractValueInFormulaFeature(Derivation deriv) {
|
||||
if (!opts.featureDomains.contains("denotation")) return;
|
||||
|
||||
if (deriv.value instanceof StringValue) {
|
||||
|
||||
//get strings from value
|
||||
List<String> valueList = new ArrayList<>();
|
||||
|
||||
String value = ((StringValue) deriv.value).value;
|
||||
|
||||
if (value.charAt(0) == '[')
|
||||
value = value.substring(1, value.length() - 1); //strip "[]"
|
||||
String[] tokens = value.split(",");
|
||||
for (String token : tokens) {
|
||||
token = token.trim(); //strip spaces
|
||||
if (token.length() > 0)
|
||||
valueList.add(token);
|
||||
}
|
||||
|
||||
//get strings from formula
|
||||
List<Formula> formulaList = deriv.formula.mapToList(formula -> {
|
||||
List<Formula> res = new ArrayList<>();
|
||||
if (formula instanceof ValueFormula) {
|
||||
res.add(formula);
|
||||
}
|
||||
return res;
|
||||
}, true);
|
||||
|
||||
for (Formula f : formulaList) {
|
||||
Value formulaValue = ((ValueFormula) f).value;
|
||||
String valueStr = (formulaValue instanceof StringValue) ? ((StringValue) formulaValue).value : formulaValue.toString();
|
||||
if (valueList.contains(valueStr))
|
||||
deriv.addFeature("denotation", "value_in_formula");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void extractRootFeatures(Example ex, Derivation deriv) {
|
||||
if (!deriv.isRootCat()) return;
|
||||
if (!opts.featureDomains.contains("root") && !opts.featureDomains.contains("root_lexical")) return;
|
||||
|
||||
List<String> derivTokens = Arrays.asList(deriv.canonicalUtterance.split("\\s+"));
|
||||
List<String> inputTokens = ex.getTokens();
|
||||
//alignment features
|
||||
BipartiteMatcher bMatcher = new BipartiteMatcher();
|
||||
List<String> filteredInputTokens = filterStopWords(inputTokens);
|
||||
List<String> filteredDerivTokens = filterStopWords(derivTokens);
|
||||
|
||||
int[] assignment = bMatcher.findMaxWeightAssignment(buildAlignmentMatrix(filteredInputTokens, filteredDerivTokens));
|
||||
|
||||
if (opts.featureDomains.contains("root")) {
|
||||
//number of unmathced words based on exact match and ppdb
|
||||
int matches = 0;
|
||||
for (int i = 0; i < filteredInputTokens.size(); ++i) {
|
||||
if (assignment[i] != i) {
|
||||
matches++;
|
||||
}
|
||||
}
|
||||
deriv.addFeature("root", "unmatched_input", filteredInputTokens.size() - matches);
|
||||
deriv.addFeature("root", "unmatched_deriv", filteredDerivTokens.size() - matches);
|
||||
if (deriv.value != null) {
|
||||
if (deriv.value instanceof ListValue) {
|
||||
ListValue list = (ListValue) deriv.value;
|
||||
deriv.addFeature("root", String.format("pos0=%s&returnType=%s", ex.posTag(0), list.values.get(0).getClass()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.featureDomains.contains("root_lexical")) {
|
||||
for (int i = 0; i < assignment.length; ++i) {
|
||||
if (assignment[i] == i) {
|
||||
if (i < filteredInputTokens.size()) {
|
||||
String inputToken = filteredInputTokens.get(i).toLowerCase();
|
||||
deriv.addFeature("root_lexical", "deleted_token=" + inputToken);
|
||||
if (!simpleLexicon.lookup(inputToken).isEmpty()) {
|
||||
deriv.addFeature("root_lexical", "deleted_entity");
|
||||
}
|
||||
}
|
||||
else {
|
||||
String derivToken = filteredDerivTokens.get(i - filteredInputTokens.size());
|
||||
deriv.addFeature("root_lexical", "deleted_token=" + derivToken);
|
||||
if (!simpleLexicon.lookup(derivToken).isEmpty()) {
|
||||
deriv.addFeature("root_lexical", "deleted_entity");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Formula> getCallFormulas(Derivation deriv) {
|
||||
return deriv.formula.mapToList(formula -> {
|
||||
List<Formula> res = new ArrayList<>();
|
||||
if (formula instanceof CallFormula) {
|
||||
res.add(((CallFormula) formula).func);
|
||||
}
|
||||
return res;
|
||||
}, true);
|
||||
}
|
||||
private void extractLogicalFormFeatures(Example ex, Derivation deriv) {
|
||||
if (!opts.featureDomains.contains("lf")) return;
|
||||
for (int i = 0; i < ex.numTokens(); ++i) {
|
||||
List<Formula> callFormulas = getCallFormulas(deriv);
|
||||
if (ex.posTag(i).equals("JJS")) {
|
||||
if (ex.token(i).equals("least") || ex.token(i).equals("most")) //at least and at most are not what we want
|
||||
continue;
|
||||
for (Formula callFormula: callFormulas) {
|
||||
String callFormulaDesc = callFormula.toString();
|
||||
//LogInfo.logs("SUPER: utterance=%s, formula=%s", ex.utterance, deriv.formula);
|
||||
deriv.addFeature("lf", callFormulaDesc + "& superlative");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!opts.featureDomains.contains("simpleworld")) return;
|
||||
//specific handling of simple world methods
|
||||
if (deriv.formula instanceof CallFormula) {
|
||||
CallFormula callFormula = (CallFormula) deriv.formula;
|
||||
String desc = callFormula.func.toString();
|
||||
switch (desc) {
|
||||
case "edu.stanford.nlp.sempre.overnight.SimpleWorld.filter":
|
||||
deriv.addFeature("simpleworld", "filter&" + callFormula.args.get(1));
|
||||
break;
|
||||
case "edu.stanford.nlp.sempre.overnight.SimpleWorld.getProperty":
|
||||
deriv.addFeature("simpleworld", "getProperty&" + callFormula.args.get(1));
|
||||
break;
|
||||
case "edu.stanford.nlp.sempre.overnight.SimpleWorld.superlative":
|
||||
deriv.addFeature("simpleworld", "superlative&" + callFormula.args.get(1) + "&" + callFormula.args.get(2));
|
||||
break;
|
||||
case "edu.stanford.nlp.sempre.overnight.SimpleWorld.countSuperlative":
|
||||
deriv.addFeature("simpleworld", "countSuperlative&" + callFormula.args.get(1) + "&" + callFormula.args.get(2));
|
||||
break;
|
||||
case "edu.stanford.nlp.sempre.overnight.SimpleWorld.countComparative":
|
||||
deriv.addFeature("simpleworld", "countComparative&" + callFormula.args.get(2) + "&" + callFormula.args.get(1));
|
||||
break;
|
||||
case "edu.stanford.nlp.sempre.overnight.SimpleWorld.aggregate":
|
||||
deriv.addFeature("simpleworld", "countComparative&" + callFormula.args.get(0));
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void extractPhraseAlignmentFeatures(Example ex, Derivation deriv) {
|
||||
|
||||
if (!opts.featureDomains.contains("alignment")) return;
|
||||
if (phraseTable == null) phraseTable = loadPhraseTable();
|
||||
|
||||
//get the tokens
|
||||
List<String> derivTokens = Arrays.asList(deriv.canonicalUtterance.split("\\s+"));
|
||||
Set<String> inputSubspans = ex.languageInfo.getLowerCasedSpans();
|
||||
|
||||
for (int i = 0; i < derivTokens.size(); ++i) {
|
||||
for (int j = i + 1; j <= derivTokens.size() && j <= i + 4; ++j) {
|
||||
|
||||
String lhs = Joiner.on(' ').join(derivTokens.subList(i, j));
|
||||
if (entities.contains(lhs)) continue; //optimization
|
||||
|
||||
if (phraseTable.containsKey(lhs)) {
|
||||
Map<String, Double> rhsCandidates = phraseTable.get(lhs);
|
||||
Set<String> intersection = Sets.intersection(rhsCandidates.keySet(), inputSubspans);
|
||||
for (String rhs: intersection) {
|
||||
addAndFilterLexicalFeature(deriv, "alignment", rhs, lhs);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Map<String, Double>> loadPhraseTable() {
|
||||
Map<String, Map<String, Double>> res = new HashMap<>();
|
||||
int num = 0;
|
||||
for (String line : IOUtils.readLines(opts.phraseAlignmentPath)) {
|
||||
String[] tokens = line.split("\t");
|
||||
if (tokens.length != 3) throw new RuntimeException("Bad alignment line: " + line);
|
||||
MapUtils.putIfAbsent(res, tokens[0], new HashMap<>());
|
||||
|
||||
double value = Double.parseDouble(tokens[2]);
|
||||
if (value >= opts.phraseTableThreshold) {
|
||||
res.get(tokens[0]).put(tokens[1], value);
|
||||
num++;
|
||||
}
|
||||
}
|
||||
LogInfo.logs("Number of entries=%s", num);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
private void addAndFilterLexicalFeature(Derivation deriv, String domain, String str1, String str2) {
|
||||
|
||||
String[] str1Tokens = str1.split("\\s+");
|
||||
String[] str2Tokens = str2.split("\\s+");
|
||||
for (String str1Token: str1Tokens)
|
||||
if (entities.contains(str1Token)) return;
|
||||
for (String str2Token: str2Tokens)
|
||||
if (entities.contains(str2Token)) return;
|
||||
|
||||
if (stopWords.contains(str1) || stopWords.contains(str2)) return;
|
||||
deriv.addFeature(domain, str1 + "--" + str2);
|
||||
}
|
||||
|
||||
private void extractLexicalFeatures(Example ex, Derivation deriv) {
|
||||
|
||||
if (!opts.featureDomains.contains("lexical")) return;
|
||||
|
||||
List<String> derivTokens = Arrays.asList(deriv.canonicalUtterance.split("\\s+"));
|
||||
List<String> inputTokens = ex.getTokens();
|
||||
//alignment features
|
||||
BipartiteMatcher bMatcher = new BipartiteMatcher();
|
||||
List<String> filteredInputTokens = filterStopWords(inputTokens);
|
||||
List<String> filteredDerivTokens = filterStopWords(derivTokens);
|
||||
|
||||
double[][] alignmentMatrix = buildLexicalAlignmentMatrix(filteredInputTokens, filteredDerivTokens);
|
||||
int[] assignment = bMatcher.findMaxWeightAssignment(alignmentMatrix);
|
||||
for (int i = 0; i < filteredInputTokens.size(); ++i) {
|
||||
if (assignment[i] != i) {
|
||||
int derivIndex = assignment[i] - filteredInputTokens.size();
|
||||
String inputToken = filteredInputTokens.get(i).toLowerCase();
|
||||
|
||||
if (entities.contains(inputToken)) continue; //optimization - stop here
|
||||
|
||||
String derivToken = filteredDerivTokens.get(derivIndex).toLowerCase();
|
||||
if (!inputToken.equals(derivToken)) {
|
||||
addAndFilterLexicalFeature(deriv, "lexical", inputToken, derivToken);
|
||||
extractStringSimilarityFeatures(deriv, inputToken, derivToken);
|
||||
|
||||
//2:2 features
|
||||
if (i < filteredInputTokens.size() - 1) {
|
||||
if (assignment[i + 1] == assignment[i] + 1) {
|
||||
String inputBigram = Joiner.on(' ').join(inputToken, filteredInputTokens.get(i + 1)).toLowerCase();
|
||||
String derivBigram = Joiner.on(' ').join(derivToken, filteredDerivTokens.get(derivIndex + 1)).toLowerCase();
|
||||
if (!inputBigram.equals(derivBigram)) {
|
||||
addAndFilterLexicalFeature(deriv, "lexical", inputBigram, derivBigram);
|
||||
}
|
||||
}
|
||||
}
|
||||
//1:2 features
|
||||
if (derivIndex > 0) {
|
||||
addAndFilterLexicalFeature(deriv, "lexical", inputToken,
|
||||
Joiner.on(' ').join(filteredDerivTokens.get(derivIndex - 1), filteredDerivTokens.get(derivIndex)));
|
||||
}
|
||||
if (derivIndex < filteredDerivTokens.size() - 1) {
|
||||
addAndFilterLexicalFeature(deriv, "lexical", inputToken,
|
||||
Joiner.on(' ').join(filteredDerivTokens.get(derivIndex), filteredDerivTokens.get(derivIndex + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void extractStringSimilarityFeatures(Derivation deriv, String inputToken, String derivToken) {
|
||||
if (inputToken.startsWith(derivToken) || derivToken.startsWith(inputToken))
|
||||
deriv.addFeature("lexical", "starts_with");
|
||||
else if (inputToken.length() > 4 && derivToken.length() > 4) {
|
||||
if (inputToken.substring(0, 4).equals(derivToken.substring(0, 4)))
|
||||
deriv.addFeature("lexical", "common_prefix");
|
||||
}
|
||||
}
|
||||
|
||||
//return a list without wtop words
|
||||
private List<String> filterStopWords(List<String> tokens) {
|
||||
List<String> res = new ArrayList<>();
|
||||
for (String token : tokens) {
|
||||
if (!stopWords.contains(token))
|
||||
res.add(token);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private double[][] buildAlignmentMatrix(List<String> inputTokens, List<String> derivTokens) {
|
||||
|
||||
double[][] res = new double[inputTokens.size() + derivTokens.size()][inputTokens.size() + derivTokens.size()];
|
||||
for (int i = 0; i < inputTokens.size(); ++i) {
|
||||
for (int j = 0; j < derivTokens.size(); ++j) {
|
||||
String inputToken = inputTokens.get(i);
|
||||
String derivToken = derivTokens.get(j);
|
||||
|
||||
if (computeMatch(inputToken, derivToken) > 0d) {
|
||||
res[i][inputTokens.size() + j] = 1d;
|
||||
res[inputTokens.size() + j][i] = 1d;
|
||||
}
|
||||
else if (computeParaphrase(inputToken, derivToken) > 0d) {
|
||||
res[i][inputTokens.size() + j] = 0.5d;
|
||||
res[inputTokens.size() + j][i] = 0.5d;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < res.length - 1; i++) {
|
||||
for (int j = i + 1; j < res.length; j++) {
|
||||
if (i != j && res[i][j] < 1) {
|
||||
res[i][j] = Double.NEGATIVE_INFINITY;
|
||||
res[j][i] = Double.NEGATIVE_INFINITY;
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private double[][] buildLexicalAlignmentMatrix(List<String> inputTokens, List<String> derivTokens) {
|
||||
if (aligner == null) {
|
||||
aligner = Aligner.read(opts.wordAlignmentPath);
|
||||
}
|
||||
|
||||
double[][] res = new double[inputTokens.size() + derivTokens.size()][inputTokens.size() + derivTokens.size()];
|
||||
//init with -infnty and low score on the diagonal
|
||||
for (int i = 0; i < res.length - 1; i++) {
|
||||
for (int j = i; j < res.length; j++) {
|
||||
if (i == j) {
|
||||
res[i][j] = 0d;
|
||||
res[j][i] = 0d;
|
||||
}
|
||||
else {
|
||||
res[i][j] = -1000d;
|
||||
res[j][i] = -1000d;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < inputTokens.size(); ++i) {
|
||||
for (int j = 0; j < derivTokens.size(); ++j) {
|
||||
String inputToken = inputTokens.get(i).toLowerCase();
|
||||
String derivToken = derivTokens.get(j).toLowerCase();
|
||||
|
||||
if (computeMatch(inputToken, derivToken) > 0) {
|
||||
res[i][inputTokens.size() + j] = 1d;
|
||||
res[inputTokens.size() + j][i] = 1d;
|
||||
} else if (computeParaphrase(inputToken, derivToken) > 0) {
|
||||
res[i][inputTokens.size() + j] = 0.5d;
|
||||
res[inputTokens.size() + j][i] = 0.5d;
|
||||
}
|
||||
else if (aligner.getCondProb(inputToken, derivToken) > 0d &&
|
||||
aligner.getCondProb(derivToken, inputToken) > 0d) {
|
||||
double product = aligner.getCondProb(inputToken, derivToken) * aligner.getCondProb(derivToken, inputToken);
|
||||
res[i][inputTokens.size() + j] = product;
|
||||
res[inputTokens.size() + j][i] = product;
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// Represents a local pattern on an utterance.
|
||||
private static class Item {
|
||||
public final String tag;
|
||||
public final String data;
|
||||
public Item(String tag, String data) {
|
||||
this.tag = tag;
|
||||
this.data = data;
|
||||
}
|
||||
@Override public String toString() {
|
||||
return tag + ":" + data;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch items from the temporary state.
|
||||
// If it doesn't exist, create one.
|
||||
private static List<Item> getItems(Map<String, Object> tempState) {
|
||||
List<Item> items = (List<Item>) tempState.get("items");
|
||||
if (items == null)
|
||||
tempState.put("items", items = new ArrayList<>());
|
||||
return items;
|
||||
}
|
||||
private static void setItems(Map<String, Object> tempState, List<Item> items) {
|
||||
tempState.put("items", items);
|
||||
}
|
||||
|
||||
// TODO(yushi): make this less hacky
|
||||
private static final List<String> stopWords = Arrays.asList("\' \" `` ` \'\' a an the that which . what ? is are am be of".split(" "));
|
||||
private static final Set<String> entities =
|
||||
new HashSet<>(Arrays.asList("alice", "bob", "greenberg", "greenberg cafe", "central office",
|
||||
"sacramento", "austin", "california", "texas", "colorado", "colorado river", "red river", "lake tahoe", "tahoe", "lake huron", "huron", "mount whitney", "whitney", "mount rainier", "rainier", "death valley", "pacific ocean", "pacific",
|
||||
"sesame", "mission ave", "mission", "chelsea",
|
||||
"multivariate data analysis", "multivariate data", "data analysis", "multivariate", "data", "efron", "lakoff", "annals of statistics", "annals", "annals of", "of statistics", "statistics", "computational linguistics", "computational", "linguistics",
|
||||
"thai cafe", "pizzeria juno",
|
||||
"new york", "york", "beijing", "brown university", "ucla", "mckinsey", "google"));
|
||||
|
||||
|
||||
private static boolean isStopWord(String token) {
|
||||
return stopWords.contains(token);
|
||||
}
|
||||
|
||||
private static void populateItems(List<String> tokens, List<Item> items) {
|
||||
List<String> prunedTokens = new ArrayList<>();
|
||||
// Populate items with unpruned tokens
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
items.add(new Item("unigram", tokens.get(i)));
|
||||
if (i - 1 >= 0) {
|
||||
items.add(new Item("bigram", tokens.get(i - 1) + " " + tokens.get(i)));
|
||||
}
|
||||
if (!isStopWord(tokens.get(i)) || (i > 0 && (tokens.get(i - 1).equals('`') || tokens.get(i - 1).equals("``"))))
|
||||
prunedTokens.add(tokens.get(i));
|
||||
}
|
||||
|
||||
// Populate items with skip words removed
|
||||
for (int i = 1; i < prunedTokens.size(); i++) {
|
||||
items.add(new Item("skip-bigram", prunedTokens.get(i - i) + " " + prunedTokens.get(i)));
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the items for the input utterance.
|
||||
private static List<Item> computeInputItems(Example ex) {
|
||||
List<Item> items = getItems(ex.getTempState());
|
||||
if (items.size() != 0) return items;
|
||||
List<String> tokens = new ArrayList<>(ex.getTokens());
|
||||
populateItems(tokens, items);
|
||||
LogInfo.logs("input %s, items %s", ex, items);
|
||||
return items;
|
||||
}
|
||||
|
||||
// Return the set of tokens (partial canonical utterance) produced by the
|
||||
// derivation.
|
||||
public static List<String> extractTokens(Example ex, Derivation deriv, List<String> tokens) {
|
||||
int childIndex = 0;
|
||||
if (deriv.rule.rhs != null) {
|
||||
for (String p : deriv.rule.rhs)
|
||||
if (Rule.isCat(p))
|
||||
extractTokens(ex, deriv.children.get(childIndex++), tokens);
|
||||
else
|
||||
tokens.add(p);
|
||||
|
||||
} else if (deriv.start != -1 && deriv.end != -1) {
|
||||
for (int i = deriv.start; i < deriv.end; i++)
|
||||
tokens.add(ex.token(i));
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
// Compute the items for a partial canonical utterance.
|
||||
private static List<Item> computeCandidateItems(Example ex, Derivation deriv) {
|
||||
// Get tokens
|
||||
List<String> tokens = new ArrayList<>();
|
||||
extractTokens(ex, deriv, tokens);
|
||||
// Compute items
|
||||
List<Item> items = new ArrayList<>();
|
||||
populateItems(tokens, items);
|
||||
return items;
|
||||
}
|
||||
|
||||
private static void subtractDescendentsFeatures(Derivation deriv, Derivation subderiv) {
|
||||
if (subderiv.children != null) {
|
||||
for (Derivation child : subderiv.children) {
|
||||
deriv.getLocalFeatureVector().add(-1, child.getLocalFeatureVector());
|
||||
subtractDescendentsFeatures(deriv, child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return the "complexity" of the given derivation.
|
||||
private static int derivationSize(Derivation deriv) {
|
||||
int sum = 0;
|
||||
if (opts.countIntermediate || !(deriv.rule.lhs.contains("Intermediate"))) sum++;
|
||||
if (deriv.children != null) {
|
||||
for (Derivation child : deriv.children)
|
||||
sum += derivationSize(child);
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
private static double computeMatch(String a, String b) {
|
||||
if (a.equals(b)) return 1;
|
||||
if (LanguageInfo.LanguageUtils.stem(a).equals(LanguageInfo.LanguageUtils.stem(b))) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static double computeParaphrase(String a, String b) {
|
||||
if (computeMatch(a, b) > 0) return 0;
|
||||
|
||||
String[] aGrams = a.split(" ");
|
||||
String[] bGrams = b.split(" ");
|
||||
if (aGrams.length != bGrams.length) return 0;
|
||||
|
||||
PPDBModel model = PPDBModel.getSingleton();
|
||||
int numPpdb = 0;
|
||||
int numMisses = 0;
|
||||
for (int i = 0; i < aGrams.length; i++) {
|
||||
if (computeMatch(aGrams[i], bGrams[i]) == 0d) {
|
||||
if (model.get(aGrams[i], bGrams[i]) > 0d ||
|
||||
model.get(LanguageInfo.LanguageUtils.stem(aGrams[i]),
|
||||
LanguageInfo.LanguageUtils.stem(bGrams[i])) > 0d) {
|
||||
numPpdb++;
|
||||
}
|
||||
else {
|
||||
numMisses++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return (numMisses == 0 && numPpdb <= 1 ? 1d : 0d);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue