natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pre-release of SEMPRE 2.0

This commit is contained in:
Percy Liang 2014-12-21 18:43:16 -08:00
parent 6782fdb66b
commit 98ea9bd6a9
308 changed files with 20414 additions and 15628 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -1,9 +0,0 @@
lib
state
test-output
*.cache
virtuoso-opensource
data/free917*.json
data/emnlp*.grammar

1237
DOCUMENTATION.md Normal file
View File

File diff suppressed because it is too large Load Diff

43
Makefile
View File

@ -1,17 +1,38 @@
NAME=sempre
BUILD_DEPS = libsempre/sempre-core.jar \
libsempre/sempre-cache.jar \
libsempre/sempre-freebase.jar \
libsempre/sempre-fbalignment.jar \
libsempre/sempre-paraphrase.jar \
libsempre/sempre-corenlp.jar \
libsempre/sempre-jungle.jar
default: $(NAME).jar
DEPS := $(shell ls lib/*.jar) $(shell find src -name "*.java")
default: module-classes $(BUILD_DEPS)
classes: $(DEPS)
mkdir -p classes
javac -d classes -cp 'lib/*' -Xlint:all `find src -name "*.java"`
touch classes
module-classes:
scripts/extract-module-classes.rb
core: libsempre/sempre-core.jar
libsempre/sempre-core.jar: \
$(shell ls src/edu/stanford/nlp/sempre/*.java) \
$(shell ls src/edu/stanford/nlp/sempre/test/*.java)
cd src/edu/stanford/nlp/sempre && ant compile
cache: libsempre/sempre-cache.jar
libsempre/sempre-cache.jar: \
$(shell find src/edu/stanford/nlp/sempre/cache -name "*.java")
cd src/edu/stanford/nlp/sempre/cache && ant compile
corenlp: libsempre/sempre-corenlp.jar
libsempre/sempre-corenlp.jar: libsempre/sempre-core.jar libsempre/sempre-cache.jar \
$(shell find src/edu/stanford/nlp/sempre/corenlp -name "*.java")
cd src/edu/stanford/nlp/sempre/corenlp && ant compile
freebase: libsempre/sempre-freebase.jar
libsempre/sempre-freebase.jar: libsempre/sempre-core.jar libsempre/sempre-cache.jar \
$(shell find src/edu/stanford/nlp/sempre/freebase -name "*.java")
cd src/edu/stanford/nlp/sempre/freebase && ant compile
$(NAME).jar: classes
jar cf $(NAME).jar -C classes .
jar uf $(NAME).jar -C src .
clean:
rm -rf classes $(NAME).jar
rm -rf classes libsempre

93
QUICKSTART.md
View File

@ -1,93 +0,0 @@
This is a quickstart guide for recreating the EMNLP 2013 or ACL 2014 system.
# Download the Dependencies
These commands will download necessary resources (choose emnlp2013 or acl2014 or both):
./download-dependencies core
./download-dependencies emnlp2013
./download-dependencies acl2014
./download-dependencies fullfreebase_vdb
See `README.md` for other requirements SEMPRE depends on
(JDK 7 and Ruby).
# Install the Database
Freebase is stored in a database called virtuoso. These commands will
install a copy of it. Make sure to execute the `git checkout tags/v7.0.0`
to ensure you have a compatible version of virtuoso (instead of the most
recent version).
# For Ubuntu, make sure these dependencies are installed
sudo apt-get install -y automake gawk gperf libtool bison flex libssl-dev
# Clone into sempre folder
git clone https://github.com/openlink/virtuoso-opensource
cd virtuoso-opensource
git checkout tags/v7.0.0
./autogen.sh
./configure --prefix=$PWD/install
make
make install
cd ..
# Start the Database
This will start the virtuoso database on `localhost:3093` and import freebase:
./scripts/virtuoso start lib/freebase/93.exec/vdb 3093
# Running the System on New Questions
First make sure you have compiled sempre by running `make`.
Create a file called `testinput` that has your test questions in this format:
(example (utterance "what states make up the midwest us?") (targetValues (description "")))
(example (utterance "what is the capital of france?") (targetValues (description "Paris")))
Then run this command to test the default trained system on those two examples:
./sempre @mode=train \
@domain=webquestions \
@sparqlserver=localhost:3093 \
@cacheserver=local \
-Learner.maxTrainIters 0 \
-Dataset.inPaths test:testinput \
-Builder.inParamsPath lib/models/15.exec/params \
-Grammar.inPaths lib/models/15.exec/grammar \
-Dataset.readLispTreeFormat true
This run should take about a minute or two. This will save the output to
`state/execs/$N.exec/log` where `$N` is some number. The current system should
get the first example wrong and the second one correct.
Alternatively, you can launch an interactive shell to test out the system:
./sempre @mode=interact \
@domain=webquestions \
@sparqlserver=localhost:3093 \
@cacheserver=local \
@load=15\
@executeTopOnly=0
# Training the EMNLP 2013/ACL 2014 System
This command will train the EMNLP 2013 system on the WebQuestions dataset. It takes
a little over three days to complete.
./sempre @mode=train \
@sparqlserver=localhost:3093 \
@domain=webquestions \
@cacheserver=local
This command will train the ACL 2014 system on the WebQuestions dataset. It takes
about 24 hours to complete.
./parasempre @mode=train \
@sparqlserver=localhost:3093 \
@domain=webquestions \
@cacheserver=local
Alternatively, you can sanity check the system on the Free917 dataset by
setting `@domain=free917`. This should about an hour to complete.

157
README.md
View File

@ -1,10 +1,42 @@
# SEMPRE: Semantic Parsing with Execution
# SEMPRE 2.0: Semantic Parsing with Execution
SEMPRE is a toolkit for training semantic parsers, which map natural language
utterances to denotations (answers) via intermediate logical forms. See
TUTORIAL.md for a walkthrough of the system.
## What is semantic parsing?
If you use this system, please cite:
A semantic parser maps natural language utterances into an intermediate logical
form, which is "executed" to produce a denotation that is useful for some task.
A simple arithmetic task:
- Utterance: *What is three plus four?*
- Logical form: `(+ 3 4)`
- Denotation: `7`
A question answering task:
- Utterance: *Where was Obama born?*
- Logical form: `(place_of_birth barack_obama)`
- Denotation: `Honolulu`
A virtual travel agent task:
- Utterance: *Show me flights to Montreal leaving tomorrow.*
- Logical form: `(and (type flight) (destination montreal) (departure_date 2014.12.09))`
- Denotation: `(list ...)`
By parsing utterances into logical forms, we obtain a rich representation that
enables mucher deeper, context-aware understanding beyond the words. With the
rise of natural language interfaces, semantic parsers are becoming increasingly
more powerful and useful.
## What is SEMPRE?
SEMPRE is a toolkit that makes it easy to develop semantic parsers for new
tasks. The main paradigm is to learn a feature-rich discriminative semantic
parser from a set of utterance-denotation pairs. One can also quickly
prototype rule-based systems, learn from other forms of supervision, and
combine any of the above.
If you use SEMPRE in your work, please cite:
@inproceedings{berant2013freebase,
author = {J. Berant and A. Chou and R. Frostig and P. Liang},
@ -13,41 +45,108 @@ If you use this system, please cite:
year = {2013},
}
# Requirements
SEMPRE has been used in the following papers:
SEMPRE depends on the following:
- J. Berant and A. Chou and R. Frostig and P. Liang. [Semantic parsing on
Freebase from question-answer
pairs](http://cs.stanford.edu/~pliang/papers/freebase-emnlp2013.pdf). EMNLP,
2013.
This paper introduced SEMPRE 1.0, applied it to question answering on
Freebase, and created the WebQuestions dataset. The paper focuses on scaling
up semantic parsing via alignment and bridging, and does not talk about the
SEMPRE framework at all. To reproduce those results, check out SEMPRE 1.0.
- J. Berant and P. Liang. [Semantic Parsing via
Paraphrasing](http://cs.stanford.edu/~pliang/papers/paraphrasing-acl2014.pdf).
ACL, 2014.
This paper also used SEMPRE 1.0. The paraphrasing model is somewhat of a
offshoot, and does not use many of the core learning and parsing utiltiies in
SEMPRE. To reproduce those results, check out SEMPRE 1.0.
* Java 7
* Ruby (version 1.8.7 or 1.9)
* [fig](https://github.com/percyliang/fig)
* [Google Guava](https://code.google.com/p/guava-libraries/) (version 14)
* [Jackson JSON Processor](http://wiki.fasterxml.com/JacksonHome) (version 2.2)
* [TestNG](http://testng.org/) (version 6.8)
* [Apache Lucene](http://lucene.apache.org/) (version 4.4)
* [Stanford CoreNLP](http://nlp.stanford.edu/software/corenlp.shtml) (version 3.2)
## Where do I go next?
Aside from Java and Ruby, remaining software dependencies are among
what's fetched by the `download-depenencies` script.
- If you're new to semantic parsing, you can learn more from the [background
reading section of the
tutorial](http://github.com/percyliang/sempre/blob/master/TUTORIAL.md).
- Install SEMPRE using the instructions under **Installation** below.
- Walk through the
[tutorial](http://github.com/percyliang/sempre/blob/master/TUTORIAL.md)
to get a hands-on introduction to semantic parsing through SEMPRE.
- Read the complete
[documentation](http://github.com/percyliang/sempre/blob/master/DOCUMENTATION.md)
to learn about the different components in SEMPRE.
# Tests
# Installation
To troubleshoot or check for repository health, you can run a suite of unit
tests that come packaged with the system. The command for this is:
## Requirements
./sempre @mode=test -excludegroups xfail
You must have the following already installed on your system.
These tests assume you have a connection to a SPARQL endpoint at localhost:3093.
To avoid this assumption, you can exclude any tests that require a SPARQL
server by instead running:
- Java 8 (not 7)
- Ant 1.8.2
- Ruby 1.8.7 or 1.9
./sempre @mode=test -excludegroups xfail,sparql
Other dependencies will be downloaded as you need them. SEMPRE has been tested
on Ubuntu Linux 12.04 and MacOS X. Your mileage will vary depending on how
similar your system is.
And if you don't have emnlp2013 dependencies downloaded, you'll need to exclude
yet another group of unit tests:
## Easy setup
./sempre @mode=test -excludegroups xfail,sparql,emnlp2013
1. Clone the GitHub repository:
All of these tests should pass.
git clone https://github.com/percyliang/sempre
2. Download the minimal core dependencies (all dependencies will be placed in `lib`):
./pull-dependencies core
3. Compile the source code (this produces `libsempre/sempre-core.jar`):
make core
4. Run an interactive shell:
./run @mode=simple
You should be able to type the following into the shell and get the answer `(number 7)`:
(execute (call + (number 3) (number 4)))
To go further, check out the
[tutorial](http://github.com/percyliang/sempre/blob/master/TUTORIAL.md).
## Virtuoso graph database
If you will be using natural language to query databases (e.g., Freebase), then
you will also need to setup your own Virtuoso database (unless someone already
has done this for you):
# For Ubuntu, make sure these dependencies are installed
sudo apt-get install -y automake gawk gperf libtool bison flex libssl-dev
# Clone the repository
git clone https://github.com/openlink/virtuoso-opensource
cd virtuoso-opensource
git checkout tags/v7.0.0
# Configure
./autogen.sh
mv INSTALL INSTALL.txt # Avoid conflict on case-insensitive file systems
./configure --prefix=$PWD/install
# Make (this takes a while)
make
make install
cd ..
# ChangeLog
Changes from SEMPRE 1.0 to SEMPRE 2.0:
- Updated tutorial and documentation.
- Refactored into a core part for building semantic parsers in general;
interacting with Freebase and Stanford CoreNLP are just different modules.
- Removed fbalignment (EMNLP 2013) and paraphrase (ACL 2014) components to
avoid confusion. If you want to reproduce those systems, use SEMPRE 1.0.
# License

743
TUTORIAL.md
View File

@ -1,50 +1,59 @@
# SEMPRE: Semantic Parsing with Execution
# SEMPRE 2.0 tutorial
In this tutorial, we will provide a brief tour of SEMPRE. This tutorial is
very much about the mechanics of the system, not about the linguistics or
semantic parsing from a research point of view (for those, see the recommended
readings at the end of this document).
readings at the end of this document). For the full documentation, see
[DOCUMENTATION.md](https://github.com/percyliang/sempre/blob/master/DOCUMENTATION.md).
We will construct a semantic parser to understand a toy subset of natural
language. Concretely, the system will have the following behavior:
language. Concretely, the system we will build will have the following
behavior:
- Input: *What is three plus four?*
- Output: 7
In semantic parsing, *natural language utterances* are mapped into *logical
forms* (think programs), which are executed to produce some *denotation* (think
return value).
Recall that in semantic parsing, *natural language utterances* are mapped into
*logical forms* (think programs), which are executed to produce some
*denotation* (think return value).
## Configuration
We have assumed you have already downloaded SEMPRE and can open up a shell:
To download all the dependencies for the system, run:
./download-dependencies core
To compile the system, run:
make
To run the system, run:
java -Xmx3g -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor JavaExecutor -interactive
./run @mode=simple
This will put you in an interactive prompt where you can develop a system and
parse utterances into tiny Java programs (hence `JavaExecutor`). Note: you
might find it convenient to use `rlwrap` to get readline support.
parse utterances into tiny Java programs. Note: you might find it convenient
to use `rlwrap` to get readline support.
## Formulas and denotations
Just to provide a bit of transparency: The `run` script simply creates a
shell command and executes it. To see which command is run, do:
A logical form (`Formula`) is a hierarchical expression. For example, here are
some primitive logical forms:
./run @mode=simple -n
This should print out:
rlwrap java -cp libsempre/*:lib/* -ea edu.stanford.nlp.sempre.Main -interactive
You can pass in additional options:
./run @mode=simple -Parser.verbose 3 # Turn on more verbose debugging for the parser
./run @mode=simple -help # Shows all options and default values
## Section 1: Logical forms and denotations
A **logical form** (class `Formula` in SEMPRE) is a hierarchical expression.
In the base case, we have primitive logical forms for representing concrete
values (booleans, numbers, strings, dates, names, and lists):
(boolean true)
(number 3)
(string hello)
(date 2013 7 28)
(string "hello world")
(date 2014 12 8)
fb:en.barack_obama
(list (number 1) (number 2))
Logical forms can be constructed recursively using `call`, which takes a
function name followed by arguments:
function name followed by arguments, which are themselves logical forms:
(call + (number 3) (number 4))
(call java.lang.Math.cos (number 0))
@ -52,36 +61,68 @@ function name followed by arguments:
(call .substring (string "what is this?") (number 5) (number 7))
(call if (call < (number 3) (number 4)) (string yes) (string no))
Note that each logical form is painfully explicit about types. You would
probably not want to program directly in this language, but that is not the
point; we will generate these logical forms automatically from natural
language.
In general:
We can execute these logical forms by typing into the interactive prompt:
(call <function-name> <logical-form-argument-1> ... <logical-form-argument-n>)
Later, we will see other ways (besides `call`) of building more complex logical
forms from simpler logical forms.
Note that each logical form is painfully explicit about types. You would
probably not want to program directly in this language (baby Java using
LISP-like notation), but that is not the point; we will later generate these
logical forms automatically from natural language.
We can execute a logical form by typing the following into the interactive
prompt:
(execute (call + (number 3) (number 4)))
This should print out `(number 7)`, which we refer to as the *denotation* of
the logical form. Try to execute the other logical forms and see what you get.
In general:
(execute <logical-form>)
This should print out `(number 7)`, which we refer to as the *denotation*
(class `Value` in SEMPRE) of the logical form. Try to execute the other
logical forms and see what you get.
**Exercise 1.1**: write a logical form that computes the first word
("compositionality") in the string "compositionality is key".
### Lambda expressions
So far, we have been representing monolithic logical forms. When we start
doing parsing, it will be convenient to be able to refer to logical forms with
some of its parts abstracted out. For example, the following logical form
represents a function that takes a number and returns its square:
So far, we have been representing logical forms that produce a single output
value (e.g., "compositionality"). But one of the key ideas of having programs
(logical forms) is the power of abstraction &mdash; that we can represents
*functions* that compute an output value for each input value.
For example, the following logical form denotes a function that takes a number
and returns its square:
(lambda x (call * (var x) (var x)))
We apply a lambda expression in the usual Lispy way:
If you execute this logical form directly, you will get an error, because the
denotation of this logical form is a function, which is not handled by the
`JavaExecutor`. However, we can apply this function to an argument `(number
3)`:
((lambda x (call * (var x) (var x))) (number 3))
Executing this expression should yield `(number 9)`.
This logical form now denotes a number. Executing this logical form should
yield `(number 9)`.
Technical note: these lambda expressions are just doing macro substitution, not
actually representing higher-order functions; since there are no side effects
here, there is no difference.
In general:
(<function-logical-form> <argument-logical-form>)
**Exercise 1.2**: Adapt your logical form form Exercise 1.1 to compute the
first word of any string. Your answer should be `(lambda x ...)`. Create a
logical form that applies this on the argument `(string "compositionality is
key")`.
Technical note: these lambda expressions are actually just doing macro
substitution, not actually representing higher-order functions; since there are
no side effects here, there is no difference.
This concludes the section on logical forms and denotations. We have presented
one system of logical forms, which are executed using `JavaExecutor`. The
@ -89,30 +130,32 @@ system supports other types of logical forms, for example, those which encode
SPARQL database queries for question answering (we will get to that later).
Note that there is no mention of natural language yet...
## Parsing
## Section 2: Parsing utterances to logical forms
Having established the nature of logical forms and their denotations, let us
turn to the problem of mapping a natural language utterance into a logical
form. We will proceed by defining a *grammar*, which is a set of rules
which specify how to perform this mapping piece by piece. This is a grammar
in the computer science sense, not in the linguistic sense (again, SEMPRE
just provides a framework; you can use whatever grammar you would like).
form. Again, the key framework is *compositionality*, which roughly says that
the meaning of a full sentence is created by combining the meanings of its
parts. For us, meanings are represented by logical forms.
We will start by defining a **grammar** (class `Grammar` in SEMPRE), which is a
set of **rules** (class `Rule` in SEMPRE), which specify how to combine logical
forms to build more complex ones in a manner that is guided by the natural
language.
We will run through some examples to give you a feel for how things work,
and then go into the details. First, let us add a rule to the grammar by
entering the following into the prompt:
typing the following into the interactive prompt:
(rule $ROOT (three) (ConstantFn (number 3)))
Now to parse an utterance, just type it in:
Now to parse an utterance, just type it in to the interactive prompt:
three
Note: The first time you parse an utterance will be slower since Stanford CoreNLP
components need to be loaded into memory (e.g., the part-of-speech tagger).
The parser should print out (among other information) a line that shows that
the utterance was parsed sucessfully into a *Derivation*:
the utterance was parsed successfully into a **derivation** (class `Derivation`
in SEMPRE), which importantly carries the correct logical form `(number 3.0)`:
(derivation (formula (number 3.0)) (value (number 3.0)) (type fb:type.number))
@ -120,31 +163,32 @@ Now type in the utterance:
four
You should get no results because no rule will match `four`. To fix that, let
You should get no results because no rule matches `four`. To fix that, let
us create a more general rule:
(rule $ROOT ($PHRASE) (NumberFn))
This rule says match any phrase (sequence of tokens), pass it to a special
function called `NumberFn`, which will transform the string into a new
derivation.
This rule says for any phrase (sequence of consecutive tokens), pass it to a special
function called `NumberFn`, which will transform the phrase string into a new
derivation representing a number.
Now the system can interpret general numbers! Try typing in:
Now, you can parse the following:
twenty five million
The result should be `(number 2.5E7)`.
four
2.718
Note: if you now type in `three`, you should get two derivations that yield the
same answer, one coming from each rule.
same answer, one coming from each rule. Note that `twenty-five million` will
not parse because we are using `SimpleLanguageAnalyzer`. Later, we can using
Stanford CoreNLP to improve the basic linguistic capabilities.
So far, we have only parsed utterances using one rule, but the true power of
these grammars come from combining multiple rules. Copy and paste in the
following rules:
(rule $Expr ($PHRASE) (NumberFn))
(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))
(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))
(rule $Partial ($Operator $Expr) (JoinFn forward))
(rule $Expr ($Expr $Partial) (JoinFn backward))
(rule $ROOT ((what optional) (is optional) $Expr (? optional)) (IdentityFn))
@ -153,7 +197,7 @@ Now try typing in:
What is three plus four?
The output should be
The output should be:
(number 7)
@ -168,15 +212,15 @@ operations anywhere.
Hopefully that should give you a sense of what parsing looks like. Let us now
take a closer look. At the end of the day, a grammar declaratively specifies a
mapping from utterances to a set of candidate *Derivation*s (which are to be
scored and ranked later). A parser is an actual algorithm that takes the
grammar and generates those derivations. Recall that a derivation looks like this:
mapping from utterances to a set of candidate derivations. A **parser** (class
`Parser` in SEMPRE) is an actual algorithm that takes the grammar and generates
those derivations. Recall that a derivation looks like this:
(derivation (formula (number 3.0)) (value (number 3.0)) (type fb:type.number))
Formally, each Derivation produced by the parser has the following properties:
Formally, each derivation produced by the parser has the following properties:
1. Span i:j (e.g., 0:1): specifies the continguous portion of the input
1. Span i:j (e.g., 0:1): specifies the contiguous portion of the input
utterance (tokens i to j-1) that the Derivation is constructed from.
2. Category (e.g., `$ROOT`): categories place hard constraints on what
Derivations can be combined.
@ -203,7 +247,7 @@ There are some special categories:
Now let us see how a grammar specifies the set of derivations.
A grammar is a set of rules, and each rule has the following form:
(rule |target| (|source_1| ... |source_k|) |semantic function|)
(rule <target-category> (<source-1> ... <source-k>) <semantic-function>)
1. Target category (e.g., `$ROOT`): any derivation produced by this rule is
labeled with this category. `$ROOT` is the designated top-level category.
@ -216,10 +260,12 @@ A grammar is a set of rules, and each rule has the following form:
3. Semantic function (`SemanticFn`): a semantic function takes a sequence of
derivations corresponding to the categories in the children and produces a set
of new derivations which are to be labeled with the target category.
Semantic functions are arbitrary Java code, and allow the parser to integrate
custom logic in a flexible modular way. In the example above `ConstantFn`
is an example of a semantic function which always returns one `Derivation`
with the given logical form (e.g., `(number 3)`).
Semantic functions run arbitrary Java code, and allow the parser to integrate
custom logic in a flexible modular way. In the example above, `ConstantFn`
is an example of a semantic function which always returns one derivation
with the given logical form (e.g., `(number 3)`). `JoinFn` produces a
derivation whose logical form is the composition of the logical forms of the
two source derivations.
Derivations are built recursively: for each category and span, we construct a
set of Derivations. We can apply a rule if there is some segmentation of the
@ -228,8 +274,8 @@ with category |source_i|. In this case, we pass the list of derivations as
input into the semantic function. The output is a set of derivations (possibly
zero).
The first rule is a familiar one that just parses numbers such as `three
million` into the category `$Expr`:
The first rule is a familiar one that just parses strings such as *three*
into the category `$Expr`:
(rule $Expr ($PHRASE) (NumberFn))
@ -241,22 +287,8 @@ category `$Expr` and span 0:1. The same goes for *four* on span 2:3.
The next two rules map the tokens *plus* and *times* to a static logical form
(returned by `ConstantFn`):
(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
Note that the `ConstantFn` specifies both the logical form and the type (note
that the type is a property of the derivation, not of the logical form). Here,
the logical form is
(lambda y (lambda x (call + (var x) (var y))))
which represents a curried function that takes `y` and `x` and returns calls
`+` on the values. The type of this entry is:
(-> fb:type.number (-> fb:type.number fb:type.number))
The type is used in parsing to prevent bad derivations from being combined
spuriously.
(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))
(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))
The next two rules are the main composition rules:
@ -266,16 +298,16 @@ The next two rules are the main composition rules:
The semantic function `(JoinFn forward)` takes two a lambda term `$Operator`
and an argument `$Expr` and returns a new derivation by forward application:
Input $Operator: (lambda y (lambda x (call + (var x) (var y))))
Input $Expr: (number 4)
Output $Partial: (lambda x (call + (var x) (number 4)))
Source $Operator: (lambda y (lambda x (call + (var x) (var y))))
Source $Expr: (number 4)
Target $Partial: (lambda x (call + (var x) (number 4)))
The semantic function `(Join backward)` takes an argument `$Expr` and a lambda
term `$Partial` and returns a new derivation by backward application:
Input $Expr: (number 3)
Input $Partial: (lambda x (call + (var x) (number 4)))
Output $Expr: (call + (number 3) (number 4))
Source $Expr: (number 3)
Source $Partial: (lambda x (call + (var x) (number 4)))
Target $Expr: (call + (number 3) (number 4))
(rule $ROOT ((what optional) (is optional) $Expr (? optional)) (IdentityFn))
@ -283,80 +315,233 @@ We allow some RHS elements to be optional, so that we could have typed in
`three plus four` or `three plus four?`. `IdentityFn` simply takes the logical
form corresponding to `$Expr` and passes it up.
## Learning
The complete derivation for *three plus four* is illustrated here:
$ROOT : (call + (number 3) (number 4)))
| [IdentityFn]
$Expr : (call + (number 3) (number 4)))
| [JoinFn backward]
+-------------------------+-------------------------+
| |
| $Partial : (lambda x (call + (var x) (number 4)))
| | [JoinFn forward]
| +-----------------------------+------------------------------+
| | |
$Expr : (number 3) $Operator : (lambda y (lambda x (call + (var x) (var y)))) $Expr : (number 4)
| [NumberFn] | [ConstantFn] | [NumberFn]
$PHRASE: three | $PHRASE : four
| [built-in] | | [built-in]
three plus four
**Exercise 2.1**: write rules can parse the following utterances into
into the category `$Expr`:
length of hello world # 11
length of one # 3
Your rules should look something like:
(rule $Function (length of) ...)
(rule $Expr ($Function $PHRASE) ...)
**Exercise 2.2**: turn your "first word" program into a rule so that you can
parse the following utterances into `$String`:
first word in compositionality is key # compositionality
first word in a b c d e # a
**Exercise 2.3**: combine all the rules that you have written to produce one grammar
that can parse the following:
two times length of hello world # 22
length of hello world times two # (what happens here?)
To summarize, we have shown how to connect natural language utterances and
logical forms using grammars, which specify how one can compositionally form
the logical form incrementally starting from the words in the utterance. Note
that we are dealing with grammars in the computer science sense, not in the
linguistic sense, as we are not developing a linguistic theory of
grammaticality; we are merely trying to parse some useful subset of utterances
for some task. Given an utterance, the grammar defines an entire set of
derivations, which reflect both the intrinsic ambiguity of language as well as
the imperfection of the grammar. In the next section, we will show how to
learn a semantic parser that can resolve these ambiguities.
### Saving to a file (optional)
You can put a set of grammar rules in a file (e.g.,
`data/tutorial-arithmetic.grammar`) and load it:
./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar
If you make edit the grammar, you can reload the grammar without exiting the
prompt by typing:
(reload)
### Using CoreNLP (optional)
Recall that we were able to parse *four*, but not *twenty-five million*,
because we used the `SimpleLanguageAnalyzer`. In this section, we will show
how to leverage Stanford CoreNLP, which provides us with more sophisticated
linguistic processing on which we can build more advanced semantic parsers.
First, we need to do download an additional dependency (this could take a while
to download because it loads all of the Stanford CoreNLP models for
part-of-speech tagging, named-entity recognition, syntactic dependency parsing,
etc.):
./pull-dependencies corenlp
Compile it:
make corenlp
Now we can load the SEMPRE interactive shell with `CoreNLPAnalyzer`:
./run @mode=simple -languageAnalyzer corenlp.CoreNLPAnalyzer -Grammar.inPaths data/tutorial-arithmetic.grammar
The following utterances should work now (the initial utterance will take a few
seconds while CoreNLP models are being loaded):
twenty-five million
twenty-five million plus forty-two
## Section 3: Learning
So far, we have used the grammar to generate a set of derivations given an
utterance. In general, this set is huge and we need a way to rank the
derivations. Our strategy is to provide the system with training examples,
from which the system can learn a model that places a distribution over
derivations given utterances.
utterance. We could work really hard to make the grammar not overgenerate, but
this will in general be hard to do without tons of manual effort. So instead, we will
use machine learning to learn a model that can choose the best derivation (and
thus logical form) given this large set of candidates. So the philosophy is:
To enable learning in the interactive prompt, do the following (we could have
also passed these in as command-line arguments, e.g.,
`-Master.onlineLearnExamples true -FeatureExtractor.featureDomains rule`):
- Grammar: small set of manual rules, defines the candidate derivations
- Learning: automatically learn to pick the correct derivation using features
(set Master.onlineLearnExamples true)
(set FeatureExtractor.featureDomains rule)
In a nutshell, the learning algorithm (class `Learner` in SEMPRE) uses
stochastic gradient descent to optimize the conditional log-likelihood of the
denotations given the utterances in a training set. Let us unpack this.
The first statement turns on online learning updates (this is only necessary
for the interactive prompt). The second statement adds features that keep
track of which rules we are using.
### Components of learning
First, for each derivation, we extract a set of **features** (formally a map
from strings to doubles &mdash; 0 or 1 for indicator features) using a feature
extractor (class `FeatureExtractor` in SEMPRE), which is an arbitrary function
on the derivation. Given a parameter vector theta (class `Params` in SEMPRE),
which is also a map from strings to doubles, the inner product gives us a
score:
Score(x, d) = features(x, d) dot theta,
where x is the utterance and d is a candidate derivation.
Second, we define a **compatibility function** (class `ValueEvaluator` in SEMPRE)
between denotations, which returns a number between 0 and 1. This allows us
learn with approximate values (e.g., "3.5 meters" versus "3.6 meters") and
award partial credit.
Third, we have a dataset (class `Dataset` in SEMPRE) consisting of **examples**
(class `Example` in SEMPRE), which specifies utterance-denotation pairs.
Datasets can be loaded from files; here is what
`data/tutorial-arithmetic.grammar` looks like:
(example
(utterance "three and four")
(targetValue (number 7))
)
Intuitively, the learning algorithm will tune the parameter vector theta so
that derivations with logical forms whose denotations have high compatibility
with the target denotation are assigned higher scores. For the mathematical
details, see the learning section of this
[paper](http://www.stanford.edu/~cgpotts/manuscripts/liang-potts-semantics.pdf).
### No learning
As a simple example, imagine that a priori, we do not know what the word *and*
means: it could be either plus or times. Let us add two rules to capture the
two possibilities:
two possibilities (this is reflected in `data/tutorial-arithmetic.grammar`):
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
Now type in
Start the interactive prompt:
./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar
and type in:
three and four
There should be two derivations each with probability 0.5:
There should be two derivations each with probability 0.5 (the system arbitrarily chooses one):
(derivation (formula (((lambda y (lambda x (call + (var x) (var y)))) (number 4.0)) (number 3.0))) (value (number 7.0)) (type fb:type.number)) [score=0, prob=0.500]
(derivation (formula (((lambda y (lambda x (call * (var x) (var y)))) (number 4.0)) (number 3.0))) (value (number 12.0)) (type fb:type.number)) [score=0, prob=0.500]
(derivation (formula (((lambda y (lambda x (call + (var x) (var y)))) (number 4.0)) (number 3.0))) (value (number 7.0)) (type fb:type.number)) [score=0, prob=0.500]
You will also see the features that are active for the first derivation. For
example, the following feature represents the fact that one of the rules we
just added is active:
### Batch learning
[ rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))) ] 0 = 1 * 0
To perform (batch) learning, we run SEMPRE:
So far all the features have weight zero, so all the scores (dot products
between feature vector and weight vector) are also zero.
./run @mode=simple -Grammar.inPaths data/tutorial-arithmetic.grammar -FeatureExtractor.featureDomains rule -Dataset.inPaths train:data/tutorial-arithmetic.examples -Learner.maxTrainIters 3
We can add a training example which says that the utterance should map to the first derivation:
The `rule` feature domain tells the feature extractor to increment the feature
each time the grammar rule is applied in the derivation. `Dataset.inPaths`
specifies the examples file to train on, and `-Learner.maxTrainIters 3`
specifies that we will iterate over all the examples three times.
(accept 0)
This should perform a stochastic gradient update of the parameters. Now type in:
Now type:
three and four
The first derivation should have much higher probability (around 0.88). To see
the new parameters, type in:
The correct derivation should now have much higher score and probability:
(derivation (formula (((lambda y (lambda x (call + (var x) (var y)))) (number 4)) (number 3))) (value (number 7)) (type fb:type.any)) [score=18.664, prob=0.941]
(derivation (formula (((lambda y (lambda x (call * (var x) (var y)))) (number 4)) (number 3))) (value (number 12)) (type fb:type.any)) [score=15.898, prob=0.059]
You will also see the features that are active for the predicted derivation.
For example, the following line represents the feature indicating that we
applied the rule mapping *and* to `+`:
[ rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call + (var x) (var y))))) ] 1.383 = 1 * 1.383
The feature value is 1, the feature weight is 1.383, and their product is the
additive contribution to the score of this derivation. You can look at the score of the other derivation:
(select 1)
The corresponding feature there is:
[ rule :: $Operator -> and (ConstantFn (lambda y (lambda x (call * (var x) (var y))))) ] -1.383 = 1 * -1.383
This negative contribution to the score is why we favored the `+` derivation
over this `*` one.
We can also inspect the parameters:
(params)
This concludes the basics of logical forms, denotations, grammars, features,
and learning. In practice, much of these operations can be done offline by
putting the grammar rules in a file (`data/tutorial.grammar`) and the examples
in another file (`data/tutorial.examples.json). Then you can one run command
to perform what we just did:
### Online learning
java -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor JavaExecutor -Grammar.inPaths data/tutorial.grammar -Dataset.inPaths train:data/tutorial.examples.json -readLispTreeFormat false -trainFrac 0.5 -devFrac 0.5 -maxTrainIters 2 -featureDomains rule -execDir tutorial.out
Finally, you can also do (online) learning directly in the prompt:
This will create a directory `tutorial.out` which records all the information
associated with this run.
(accept 1)
## Lambda DCS and SPARQL
This will accept the `*` derivation as the correct one and update the
parameters on the fly. If you type:
So far, we have worked with `JavaExecutor` as the engine that maps logical
forms to denotations by executing Java code. A major application of semantic
parsing is where the logical forms are database queries. In this section, we
will look at querying graph databases.
three and four
again, you will see that the probability of the `+` derivation has decreased.
If you type `(accept 1)` a few more times, the `*` derivation will dominate
once more.
## Section 4: Lambda DCS and SPARQL
So far, we used `JavaExecutor` to map logical forms to denotations by executing
Java code. A major application of semantic parsing (and indeed the initial one
that gave birth to SEMPRE) is where the logical forms are database queries. In
this section, we will look at querying graph databases.
A graph database (e.g., Freebase) stores information about entities
and their properties; concretely, it is just a set of triples $(s, p, o)$,
@ -370,34 +555,65 @@ each triple is a directed edge between two nodes labeled with the property.
See `data/tutorial.ttl` for an example of a tiny subset of the Freebase graph
pertaining to geography about California.
We will use Virtuoso to provide the backend for querying this data. Make sure
you have Virtuoso downloaded and compiled (see QUICKSTART.md for instructions).
First, pull the dependencies needed for Freebase:
Start the server:
./pull-dependencies freebase
scripts/virtuoso start tutorial.vdb 3001
### Setting up your own Virtuoso graph database
Add the small graph to the database:
We use the graph database engine, Virtuoso, to store these triples and allow
querying. Follow these instructions if you want to create your own Virtuoso instance.
scripts/virtuoso add data/tutorial.ttl 3001
First, make sure you have Virtuoso installed (see Installation section of the
README.md).
Aside: if you want to work with a larger database:
Then start the server:
./download-dependencies geofreebase_ttl # Subset of Freebase for geography in .ttl format (not needed to run the server)
./download-dependencies geofreebase_vdb # Virtuoso index for subset of Freebase for geography
./download-dependencies fullfreebase_ttl # All of Freebase in .ttl format (not needed to run the server) [BIG FILE]
./download-dependencies fullfreebase_vdb # Virtuoso index for all of Freebase [BIG FILE]
freebase/scripts/virtuoso start tutorial.vdb 3001
Then, just replace `tutorial.vdb` with the appropriate `lib/freebase/??.exec/vdb` path.
Add a small graph to the database:
Now we are ready to make some queries. Start up the interactive prompt, now with the `SparqlExecutor`:
freebase/scripts/virtuoso add freebase/data/tutorial.ttl 3001
java -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor SparqlExecutor -endpointUrl http://localhost:3001/sparql -interactive
Now you can query the graph (this should print out three items):
SPARQL is a language for querying these graphs, but it will be convenient to
use a language more tailored for semantic parsing which is based on a mix
./run @mode=query @sparqlserver=localhost:3001 -formula '(fb:location.location.containedby fb:en.california)'
To stop the server:
freebase/scripts/virtuoso stop 3001
### Setting up a copy of Freebase
The best case is someone already installed Freebase for you and handed you a
host:port. Otherwise, to run your own copy of the Freebase graph (a
2013 snapshot), read on.
Download it (this is really big and takes a LONG time):
./pull-dependencies fullfreebase-vdb
Then you can start the server (make sure you have at least 60GB of memory):
freebase/scripts/virtuoso start lib/fb_data/93.exec/vdb 3093
### Lambda DCS
SPARQL is the standard language for querying graph databases, but it will be
convenient to use a language more tailored for semantic parsing. We will use
[lambda DCS](http://arxiv.org/pdf/1309.4408.pdf), which is based on a mix
between lambda calculus, description logic, and dependency-based compositional
semantics. The simplest formula is a single entity:
semantics (DCS).
We assume you have started the Virtuoso database:
freebase/scripts/virtuoso start tutorial.vdb 3001
Then start up a prompt:
./run @mode=simple-freebase @sparqlserver=localhost:3001
The simplest logical formula in lambda DCS is a single entity:
fb:en.california
@ -453,74 +669,10 @@ Here are the following types of logical forms:
`x` is in the set denoted by `u` and `y` is the value taken on by variable
`v`.
See `SparqlExecutorTest.java` for examples.
See `src/edu/stanford/nlp/sempre/freebase/test/SparqlExecutorTest.java` for
many more examples (which only work on the full Freebase).
### Parsing
So far, we have described the denotations of logical forms for querying a graph
database. Now we focus on parsing natural language utterances into these
logical forms.
The core challenge is at the lexical level: mapping natural language phrases
(e.g., *born in*) to logical predicates (e.g.,
`fb:people.person.place_of_birth`). First, we need to download the files that
specify this mapping (roughly, the mappings were created by aligning Freebase
and ClueWeb, as described the EMNLP 2013 paper, but you can just take these
mappings as given for now):
./download-dependencies emnlp2013
The following commands will require you having a Virtuoso instance that has all of Freebase indexed.
If you do not have this set up, run the following two commands:
./download-dependencies fullfreebase_vdb # Virtuoso index for all of Freebase [BIG FILE]
scripts/virtuoso start lib/freebase/93.exec/vdb 3093 # Starts server on localhost at port 3093
Now, restart the SEMPRE shell, pointing the SparqlExecutor to the Virtuoso
instance:
java -cp classes:lib/* edu.stanford.nlp.sempre.Main -executor SparqlExecutor -endpointUrl http://localhost:3093/sparql -interactive
Now, we can add rules with semantic function `LexiconFn`:
(rule $Entity ($PHRASE) (LexiconFn entity allowInexact)) # e.g., Ulm
(rule $Unary ($LEMMA_PHRASE) (LexiconFn unary)) # e.g., physicists
(rule $Binary ($PHRASE) (LexiconFn binary)) # e.g., born in
(rule $ROOT ($Entity) (IdentityFn))
(rule $ROOT ($Unary) (IdentityFn))
LexiconFn takes logical forms which corresponds to strings (e.g., `(string physicist)`)
and converts them into Freebase logical forms (e.g.,
`(fb:people.person.profession fb:en.physicist)`).
Type the following utterances into the prompt:
Ulm
physicists
Each utterance should return a list of candidate entities.
Now for compositionality, add the following rules, which will take two
logical forms and either perform an intersection or a join:
(rule $Set ($Unary $PartialSet) (MergeFn and))
(rule $PartialSet ($Binary $Entity) (JoinFn binary,unary unaryCanBeArg1))
(rule $ROOT ($Set) (IdentityFn))
This allows us to type in:
physicists born in Ulm
You should get many candidates, but the correct answer should appear in the list:
(derivation (formula (and (fb:people.person.profession fb:en.physicist) (!fb:location.location.people_born_here fb:en.ulm))) (value (list (name fb:en.albert_einstein "Albert Einstein"))) (type fb:people.person))
For an example of a more complex grammar, look at `data/emnlp2013.grammar`.
## Exercises
1. Convert the following natural language utterances into lambda-DCS logical forms:
**Exercise 4.1**: write lambda DCS logical forms for the following utterances:
`city with the largest area`
@ -534,17 +686,140 @@ For an example of a more complex grammar, look at `data/emnlp2013.grammar`.
`country with the most number of rivers`
You should familiarize yourself with the [Freebase schema](http://www.freebase.com/schema) to see which
predicates to use.
You should familiarize yourself with the [Freebase
schema](http://www.freebase.com/schema) to see which predicates to use.
Execute these on the full Freebase to find out the answer!
Execute these logical forms on the `geofreebase` subset to verify your answers.
### Parsing
2. Write a grammar that can parse the above utterances into a set of candidates
containing the true logical form you annotated above. Train a model (remember
to add features) so that the correct logical forms appear at the top of the
candidate list.
So far, we have described the denotations of logical forms for querying a graph
database. Now we focus on parsing natural language utterances into these
logical forms.
## Background reading
The core challenge is at the lexical level: mapping natural language phrases
(e.g., *born in*) to logical predicates (e.g.,
`fb:people.person.place_of_birth`). It is useful to distinguish between two
types of lexical items:
- Entities (e.g., `fb:en.barack_obama`): There are generally a huge number of
entities (Freebase has tens of millions). Often, string matching gets you
part of the way there (for example, *Obama* to `fb.en:barack_obama`), but
there is often quite a bit of ambiguity (Obama is also a city in Japan).
- Non-entities (e.g., `fb:people.person.place_of_birth`), which include unary
and binary predicates: There are fewer of these, but string matching is
unlikely to get you very far.
We could always add grammar rules like this:
(rule $Entity (the golden state) (ConstantFn fb:en.california))
(rule $Entity (california) (ConstantFn fb:en.california))
but grammars are supposed to be small, so this approach does not scale, so we
are not going to do this.
One way is to create a **lexicon**, which is a mapping from words to predicates
(see `freebase/data/tutorial-freebase.lexicon`), with entries like this:
{"lexeme": "california", "formula": "fb:en.california"}
{"lexeme": "the golden state", "formula": "fb:en.california"}
{"lexeme": "cities", "formula": "(fb:type.object.type fb:location.citytown)"}
{"lexeme": "towns", "formula": "(fb:type.object.type fb:location.citytown)"}
{"lexeme": "in", "formula": "fb:location.location.containedby"}
{"lexeme": "located in", "formula": "fb:location.location.containedby"}
Then we can add the following rules (see
`freebase/data/tutorial-freebase.grammar`):
(rule $Unary ($PHRASE) (SimpleLexiconFn (type fb:type.any)))
(rule $Binary ($PHRASE) (SimpleLexiconFn (type (-> fb:type.any fb:type.any))))
(rule $Set ($Unary) (IdentityFn))
(rule $Set ($Unary $Set) (MergeFn and))
(rule $Set ($Binary $Set) (JoinFn forward))
(rule $ROOT ($Set) (IdentityFn))
The `SimpleLexiconFn` looks up the phrase and returns all formulas that have the given type. To check the type, use:
(type fb:en.california) # fb:common.topic
(type fb:location.location.containedby) # (-> fb:type.any fb:type.any)
`MergeFn` takes the two (unary) logical forms |u| and |v| (in this case, coming from `$Unary` and `$Set`),
and forms the intersection logical form `(and |u| |v|)`.
`JoinFn` takes two logical forms (one binary and one unary) and returns the
logical form `(|b| |v|)`. Note that before we were using `JoinFn` as function
application. In lambda DCS, `JoinFn` produces an actual logical form that
corresponds to joining |b| and |v|. The two bear striking similarities, which
is the basis for the overloading.
Now start the interactive prompt:
./run @mode=simple-freebase @sparqlserver=localhost:3001 -Grammar.inPaths freebase/data/tutorial-freebase.grammar -SimpleLexicon.inPaths freebase/data/tutorial-freebase.lexicon
We should be able to parse the following utterances:
california
the golden state
cities in the golden state
towns located in california
In general, how does one create grammars? One good strategy is to start with a
single rule mapping the entire utterance to the final logical form. Then
decompose the rule into parts. For example, you might start with:
(rule $ROOT (cities in california) (ConstantFn (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.california))))
Then you might factor it into two pieces, in order to generalize:
(rule $ROOT (cities in $Entity) (lambda e (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby (var e)))))
(rule $Entity (california) (ConstantFn fb:en.california))
Note that in the first rule, we are writing `(lambda x ...)` directly. This
means, take the logical form for the source (`$Entity`) and substitute it in
for `x`.
We can refactor the first rule:
(rule $ROOT ($Unary in $Entity) (lambda x (and (var u) (fb:location.location.containedby (var e)))))
(rule $Unary (cities) (ConstantFn (fb:type.object.type fb:location.citytown)))
and so on...
**Exercise 4.2**: Write a grammar that can parse the utterances from Exercise
4.1 into a set of candidates containing the true logical form you annotated.
Of course you can trivially write one rule for each example, but try to
decompose the grammars as much as possible. This is what will permit
generalization.
**Exercise 4.3**: Train a model so that the correct logical forms appear at the
top of the candidate list on the training examples. Remember to add features.
## Debugging
In the beginning, SEMPRE grammars can be difficult to debug. This is primarily
because everything is dynamic, which means that minor typos result in empty
results rather than errors.
The first you should do is to check that you do not have typos. Then, try to
simplify your grammar as much as possible (comment things out) until you have
the smallest example that fails. Then you should turn on more debugging
output:
Only derivations that reach `$ROOT` over the entire span of the sentence are
built. You can also turn on debugging to print out all intermediate
derivations so that you can see where something is failing:
(set Parser.verbose 3) # or pass -Parser.verbose 3 on the command-line
Often derivations fail because an intermediate combination does not type check.
This option will print out all combinations which are tried. You might find
that you are combining two logical forms in the wrong way:
(set JoinFn.verbose 3)
(set JoinFn.showTypeCheckFailures true)
(set MergeFn.verbose 3)
(set MergeFn.showTypeCheckFailures true)
## Appendix: Background reading
So far this tutorial has provided a very operational view of semantic parsing
based on SEMPRE. The following references provide a broader look at the area

4
data/tutorial-arithmetic.examples Normal file
View File

@ -0,0 +1,4 @@
(example
(utterance "three and four")
(targetValue (number 7))
)

12
data/tutorial.grammar → data/tutorial-arithmetic.grammar
View File

@ -1,10 +1,12 @@
# Arithmetic example from TUTORIAL.md
(rule $Expr ($PHRASE) (NumberFn))
(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (plus) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))
(rule $Operator (times) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))
(rule $Partial ($Operator $Expr) (JoinFn forward))
(rule $Expr ($Expr $Partial) (JoinFn backward))
(rule $ROOT ((what optional) (is optional) $Expr (? optional)) (IdentityFn))
# Ambiguity
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y)))) (-> fb:type.number (-> fb:type.number fb:type.number))))
# Example of ambiguity to demonstrate learning
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call * (var x) (var y))))))
(rule $Operator (and) (ConstantFn (lambda y (lambda x (call + (var x) (var y))))))

11
data/tutorial.examples.json
View File

@ -1,11 +0,0 @@
[
{
"utterance": "What is seven and thirty five?",
"targetValue": "(number 42)"
},
{
"utterance": "What is ten and ten?",
"targetValue": "(number 20)"
}
]

4
data/unittest-learn.examples.json
View File

@ -1,4 +0,0 @@
[
{"utterance": "American", "targetFormula": "(fb:people.person.nationality fb:en.united_states)"},
{"utterance": "American born in Paris", "targetFormula": "(and (fb:people.person.nationality fb:en.united_states) (fb:people.person.place_of_birth fb:en.paris))"}
]

BIN
demo-www/bullet.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 966 B

1
demo-www/index.html Normal file
View File

@ -0,0 +1 @@
Hello there.

64
demo-www/main.css Normal file
View File

@ -0,0 +1,64 @@
a:link {text-decoration: none; color: 00008b}
a:visited {text-decoration: none}
a:active {text-decoration: none}
a:hover {text-decoration: none; color: red}
body {background-color: white; margin-left: 50px; font-family: sans-serif}
.question {margin: 10px; font-size: 24px}
.ask {font-size: 24px}
.answer {
font-size: 20;
/*background-image: url(bullet.png);
background-repeat: no-repeat;*/
padding-left: 24px;
}
.message {font-size: 15}
.description {margin-top: 50px; font-size: 18}
.details {
border-style: groove;
margin-top: 10px;
padding: 10px;
font-size: 13px;
margin-right: 50px
}
.listHeader {font-size:18; font-weight:bold; color:brown; white-space:nowrap}
.nowrap {white-space:nowrap}
.bubble {width:300px}
.word {font-size:16; color:darkgreen; font-style:italic}
.predicate {font-size:16; color:red}
.tag {font-size:14; color:black; white-space:nowrap}
.correctButton {color:green}
.wrongButton {color:red}
.lexicalResponse table {border-spacing:3px 0px; text-align:center}
table.predInfo td {padding:1px; text-align:left; font-size:14; white-space:nowrap}
table.context {padding:5; border:1px dashed brown}
.valueTable {border: 2px solid gray; padding: 2px}
.groupResponse td {vertical-align:top; padding:1px}
.listResponse li {font-size:12}
.error {color:red; font-weight:bold}
/* Tooltip (http://psacake.com/web/jl.asp) */
a.info {
position:relative;
z-index:24;
color:#000;
text-decoration:none
}
a.info span.tooltip {display: none}
a.info:hover {z-index:25; background-color:#ff0}
a.info:hover span.tooltip {
display:block;
position:absolute;
border:1px solid #0cf;
background-color:#eff; color:#000;
top:2em; left:2em
}

2
demo-www/main.js Normal file
View File

@ -0,0 +1,2 @@
function submit() {
}

70
download-dependencies
View File

@ -1,70 +0,0 @@
#!/usr/bin/ruby
$modules = [:core,
:emnlp2013,
:acl2014,
:geofreebase_vdb,
:geofreebase_ttl,
:fullfreebase_vdb,
:fullfreebase_ttl]
def usage
puts "Usage: ./download-dependencies <#{$modules.join('|')}>"
end
if ARGV.size == 0
usage
exit 1
end
def run(command)
puts "RUNNING: #{command}"
if not system(command)
puts "FAILED: #{command}"
exit 1
end
end
def download(release, path, baseUrl='http://nlp.stanford.edu/software/sempre')
url = baseUrl + '/release-' + release.to_s
isDirectory = (path.end_with?('.exec') or path.end_with?('vdb') or
path.end_with?('free917') or path.end_with?('inexact') or path.end_with?('prolog')) # Superficial test for directory
if release != :code and !path.end_with?('.gz', '.tgz', '.jar')
path += '.tar' if isDirectory
path += '.bz2'
end
run("mkdir -p #{File.dirname(path)}")
run("wget -c '#{url}/#{path}' -O #{path}")
if release != :code and path.end_with?('.bz2')
if isDirectory
run("cd #{File.dirname(path)} && tar xjf #{File.basename(path)}")
else
run("bzip2 -fd #{path}")
end
end
end
def downloadFromFileList(release)
files = []
File.foreach('release-' + release.to_s + '.files') { |line|
file = line.sub(/#.*$/, '').sub(/^\s*/, '').sub(/\s*$/, '')
next if file.size == 0
files << file
}
files.each { |path|
download(release, path)
}
end
def main(which)
if not $modules.include?(which)
usage
exit 1
end
downloadFromFileList(which)
end
ARGV.each { |mod|
mod = mod.to_sym
main(mod)
}

4
fig/bin/chunk
View File

@ -1,9 +1,9 @@
#!/usr/bin/ruby
require File.dirname($0)+'/../lib/myutils'
require 'pathname'
require File.dirname(Pathname.new($0).realpath) + '/../lib/myutils'
# Print out a chunk of a file. Specifically, print out the header lines.
# Use
$file, $headerNumLines, $chunkSize, $indices, $printNumChunks, $verbose = extractArgs(:spec => [
['file', String, nil, true, 'Big file to read'],
['headerNumLines', Fixnum, 0, false, 'Number of header lines to include'],

5
fig/bin/qcreate
View File

@ -83,6 +83,7 @@ cmdFile = "#{execPath}/#{id}.sh"
out = open(cmdFile, 'w')
cmd = ARGV.map{|x| x =~ /[ "]/ ? x.inspect : x}.join(' ').gsub(/_OUTPATH_/, execPath)
cmd += " > #{execPath}/stdout 2> #{execPath}/stderr" if qsub
out.puts "#!/bin/bash"
out.puts "cd #{Dir.pwd}"
out.puts cmd
out.close
@ -91,7 +92,7 @@ puts cmd
system "git rev-parse HEAD > #{execPath}/git-hash" if File.exists?('.git')
if qsub
exec("qsub #{cmdFile} -o /dev/null -e /dev/null")
exec("qsub #{File.expand_path(cmdFile)} -o /dev/null -e /dev/null")
else
exec("bash #{cmdFile}")
exec("bash #{File.expand_path(cmdFile)}")
end

3
freebase/README.md Normal file
View File

@ -0,0 +1,3 @@
This directory contains all the small data files and scripts relevant to
semantic parsing on Freebase. In particular, it contains grammars from the
EMNLP 2013 system and other examples.

127
freebase/data/emnlp2013.grammar Normal file
View File

@ -0,0 +1,127 @@
# Grammar used for the EMNLP 2013 submission.
# This grammar handles simple questions involving one entity, one binary, and
# at most one unary.
# Supports Webquestions and Free917.
############################################################
# Lexicon
### POS filter: Use POS tags to decide which spans should trigger something.
# Matching unaries (SimpleNounPhrase)
(rule $Noun ($LEMMA_TOKEN) (FilterPosTagFn token WRB WP NN NNS NNP NNPS))
(rule $SimpleNounPhrase ($Noun) (ConcatFn " "))
(rule $SimpleNounPhrase ($Noun $SimpleNounPhrase) (ConcatFn " "))
# Matching entities (NamedEntity): note - don't lemmatize
# Match an entity if it is a sequence of NE tags or NNP tags or of minimal length
(rule $NamedEntity ($PHRASE) (FilterNerSpanFn PERSON ORGANIZATION LOCATION MISC))
(rule $NamedEntity ($PHRASE) (FilterPosTagFn span NNP))
# Allow length 1 spans to trigger entities in Free917, but not in WebQuestions.
(when webquestions
(rule $TokenSpan ($PHRASE) (FilterSpanLengthFn 2))
)
(when free917
(rule $TokenSpan ($PHRASE) (FilterSpanLengthFn 1))
(rule $TokenSpan ($LEMMA_TOKEN) (FilterSpanLengthFn 1))
)
# Matching binaries (CompositeRel)
(rule $Verb ($LEMMA_TOKEN) (FilterPosTagFn token VB VBD VBN VBG VBP VBZ VBD-AUX))
(rule $Particle ($LEMMA_TOKEN) (FilterPosTagFn token RP))
(rule $Prep ($LEMMA_TOKEN) (FilterPosTagFn token IN TO))
(rule $Adj ($LEMMA_TOKEN) (FilterPosTagFn token JJ))
(rule $Rel ($LEMMA_TOKEN) (FilterPosTagFn token NN NNS NNP NNPS VB VBD VBN VBG VBP VBZ IN VBD-AUX JJ))
(rule $BaseRel ($Rel) (IdentityFn)) # parents
(rule $BaseRel ($Verb $SimpleNounPhrase) (ConcatFn " "))
(rule $BaseRel ($Verb $Particle) (ConcatFn " ")) # grow up
(rule $CompositeRel ($BaseRel) (ConcatFn " ")) # grow up, parents
(rule $CompositeRel ($BaseRel $Prep) (ConcatFn " ")) # grow up in, parents of
### Lexicon: call LexiconFn on the spans.
(when inexact
(rule $Entity ($NamedEntity) (LexiconFn entity inexact))
(rule $Entity ($TokenSpan) (LexiconFn entity inexact))
)
(when exact
(rule $Entity ($NamedEntity) (LexiconFn entity exact))
(rule $Entity ($TokenSpan) (LexiconFn entity exact))
)
(when fbsearch
(rule $Entity ($NamedEntity) (LexiconFn entity fbsearch))
(rule $Entity ($TokenSpan) (LexiconFn entity fbsearch))
)
(when fbsearch0
(rule $Entity ($NamedEntity) (LexiconFn entity fbsearch))
)
(rule $Entity ($PHRASE) (DateFn)) # Example: 2002
(rule $Unary ($SimpleNounPhrase) (LexiconFn unary)) # Example: "city"
(rule $Unary (how $Adj) (ConstantFn (fb:type.object.type fb:type.int) fb:type.int))
(rule $Unary (how $Adj) (ConstantFn (fb:type.object.type fb:type.float) fb:type.float))
(rule $Binary ($CompositeRel) (LexiconFn binary)) # Example: "capital", "developed"
#### Lexicon: hard coded unaries
(rule $Unary (who) (ConstantFn (fb:type.object.type fb:people.person) fb:people.person))
(rule $Unary (who) (ConstantFn (fb:type.object.type fb:organization.organization) fb:organization.organization))
(rule $Unary (who) (ConstantFn (fb:type.object.type fb:business.company) fb:business.company))
(rule $Unary (where) (ConstantFn (fb:type.object.type fb:organization.organization) fb:organization.organization))
(rule $Unary (where) (ConstantFn (fb:type.object.type fb:location.location) fb:location.location))
(rule $Unary (where) (ConstantFn (fb:type.object.type fb:education.educational_institution) fb:education.educational_institution))
(rule $Unary (when) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime))
(rule $Unary (date) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime))
(rule $Unary (year) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime))
(rule $Unary (day) (ConstantFn (fb:type.object.type fb:type.datetime) fb:type.datetime))
### Lexicon: hard coded binaries
(rule $Binary (,) (ConstantFn fb:location.location.containedby (-> fb:location.location fb:location.location)))
# Aggregation
(rule $CountStr (how many) (ConstantFn null null))
(rule $CountStr (number of) (ConstantFn null null))
(rule $Operator ($CountStr) (ConstantFn (lambda x (count (var x))) (-> fb:common.topic fb:type.number)))
############################################################
(rule $Padding ($PHRASE) (IdentityFn)) # Can skip these sequences
(rule $ROOT (($Padding optional) $Operator ($Padding optional) $Set) (JoinFn forward betaReduce))
(rule $ROOT (($Padding optional) $Set ($Padding optional)) (IdentityFn))
### Combine entity and binary
(when join
(rule $BaseSet ($Entity ($Padding optional) $Binary) (JoinFn unary,binary unaryCanBeArg0 unaryCanBeArg1))
(rule $BaseSet ($Binary ($Padding optional) $Entity) (JoinFn binary,unary unaryCanBeArg0 unaryCanBeArg1))
)
### Create binary out of thin air
(when bridge
(rule $BaseSet ($Unary ($Padding optional) $Entity) (BridgeFn unary headFirst))
(rule $BaseSet ($Entity ($Padding optional) $Unary) (BridgeFn unary headLast))
(rule $BaseSet ($Entity) (BridgeFn entity headLast))
)
(when inject
(rule $BaseSet ($BaseSet ($Padding optional) $Entity) (BridgeFn inject headFirst))
(rule $BaseSet ($Entity ($Padding optional) $BaseSet) (BridgeFn inject headLast))
)
(when (not compositional)
(rule $Set ($BaseSet) (IdentityFn))
(rule $Set ($Unary ($Padding optional) $BaseSet) (MergeFn and))
)
(when compositional
(rule $Set ($BaseSet) (IdentityFn))
### Combine set and binary
(when join
(rule $Set ($Binary ($Padding optional) $Set) (JoinFn binary,unary unaryCanBeArg0 unaryCanBeArg1))
(rule $Set ($Set ($Padding optional) $Binary) (JoinFn unary,binary unaryCanBeArg0 unaryCanBeArg1))
)
# Merge two sets
(rule $Set ($Set ($Padding optional) $Set) (MergeFn and))
)

278
freebase/data/free917.test.examples.canonicalized.json Normal file
View File

@ -0,0 +1,278 @@
[
{"utterance": "how many tv programs did danny devito produce", "targetFormula": "(count (!fb:tv.tv_producer_term.program ((lambda x (fb:tv.tv_producer_term.producer (var x))) fb:en.danny_devito)))"},
{"utterance": "how many countries are in south america", "targetFormula": "(count (fb:location.location.containedby fb:en.south_america))"},
{"utterance": "what product lines does ipod include", "targetFormula": "(!fb:business.product_line.includes_product_lines fb:en.ipod)"},
{"utterance": "when was starry night painted", "targetFormula": "(!fb:visual_art.artwork.date_completed fb:en.de_sterrennacht)"},
{"utterance": "when was interstate 579 formed", "targetFormula": "(!fb:transportation.road.formed fb:en.interstate_579)"},
{"utterance": "what area did the meiji constitution govern", "targetFormula": "(!fb:law.constitution.country fb:en.meiji_constitution)"},
{"utterance": "who is the lyricist for spamalot", "targetFormula": "(!fb:theater.play.lyricist fb:en.spamalot)"},
{"utterance": "what is yahoo!'s slogan", "targetFormula": "(!fb:organization.organization.slogan fb:m.019rl6)"},
{"utterance": "what is the population of belgium", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.belgium))"},
{"utterance": "who is the founder of savealot", "targetFormula": "(!fb:organization.organization.founders fb:en.save-a-lot)"},
{"utterance": "what library system is the sunset branch library in", "targetFormula": "(!fb:library.public_library.library_system fb:en.sunset_branch_library)"},
{"utterance": "where was the 3 juno asteroid discovered", "targetFormula": "(!fb:astronomy.astronomical_discovery.discovery_site fb:en.3_juno)"},
{"utterance": "what is the population of europe", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.europe))"},
{"utterance": "what weight class was the fight of the century", "targetFormula": "(!fb:boxing.boxing_match.weight_class fb:en.fight_of_the_century)"},
{"utterance": "who designed pac-man", "targetFormula": "(!fb:cvg.computer_videogame.designers fb:en.pac-man)"},
{"utterance": "when was the order of saint michael founded", "targetFormula": "(!fb:royalty.order_of_chivalry.date_founded fb:en.order_of_saint_michael)"},
{"utterance": "what is the collection of postcards called", "targetFormula": "(!fb:interests.collection_category.name_of_collection_activity fb:en.postcard)"},
{"utterance": "what year was ron glass an award nominee", "targetFormula": "(!fb:award.award_nomination.year ((lambda x (fb:award.award_nomination.award_nominee (var x))) fb:en.ron_glass))"},
{"utterance": "what's the horsepower of an alluminum-alloy v6 engine", "targetFormula": "(!fb:automotive.engine.horsepower fb:en.alluminum_alloy_v6)"},
{"utterance": "what causes prostate cancer", "targetFormula": "(!fb:medicine.disease.causes fb:en.prostate_cancer)"},
{"utterance": "when was the iphone introduced", "targetFormula": "(!fb:computer.computer.introduced fb:en.iphone)"},
{"utterance": "in what martial art does christopher adams have a black belt", "targetFormula": "(!fb:martial_arts.martial_arts_certification.art (and ((lambda x (fb:martial_arts.martial_arts_certification.qualification (var x))) fb:en.black_belt) ((lambda x (fb:martial_arts.martial_arts_certification.person (var x))) fb:en.christopher_adams)))"},
{"utterance": "who was the librettist for the magic flute", "targetFormula": "(!fb:opera.opera.librettist fb:en.the_magic_flute)"},
{"utterance": "what episode of snl did ben stiller host", "targetFormula": "(!fb:base.saturdaynightlive.snl_host.episodes_hosted fb:en.ben_stiller)"},
{"utterance": "who created the philosopher_s stone", "targetFormula": "(!fb:fictional_universe.fictional_object.created_by fb:en.philosophers_stone)"},
{"utterance": "how many organizations are in the automobile industry", "targetFormula": "(count (fb:business.business_operation.industry fb:en.automobile))"},
{"utterance": "what was procter & gamble's net profit in 1955", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:business.business_operation.net_profit (var x))) fb:en.procter_gamble))"},
{"utterance": "what is currency code for uk currency", "targetFormula": "(!fb:finance.currency.currency_code fb:en.uk)"},
{"utterance": "what's the symbol for mercury", "targetFormula": "(!fb:chemistry.chemical_element.symbol fb:m.025sw5g)"},
{"utterance": "what is jerry seinfeld religion", "targetFormula": "(!fb:people.person.religion fb:en.jerry_seinfeld)"},
{"utterance": "how big is the screen on a nikon coolpix s50", "targetFormula": "(!fb:digicams.digital_camera.lcd_screen_dimensions fb:en.nikon_coolpix_s50)"},
{"utterance": "how many speeches did winston churchill give", "targetFormula": "(count (!fb:event.speech_or_presentation.presented_work ((lambda x (fb:event.speech_or_presentation.speaker_s (var x))) fb:en.winston_churchill)))"},
{"utterance": "how many religions use the bible", "targetFormula": "(count (!fb:religion.religious_text.religious_text_of fb:en.bible))"},
{"utterance": "what is the capital of iceland", "targetFormula": "(!fb:location.country.capital fb:en.iceland)"},
{"utterance": "what public transportation is there in tokyo", "targetFormula": "(!fb:travel.travel_destination.local_transportation fb:en.tokyo)"},
{"utterance": "what is europe 's area", "targetFormula": "(!fb:location.location.area fb:en.europe)"},
{"utterance": "how many players are in the current roster of the new york mets", "targetFormula": "(count (!fb:baseball.baseball_roster_position.player ((lambda x (fb:baseball.baseball_roster_position.team (var x))) fb:en.new_york_mets)))"},
{"utterance": "what are the treatments of prostate cancer", "targetFormula": "(!fb:medicine.disease.treatments fb:en.prostate_cancer)"},
{"utterance": "where was the peseta used as currency", "targetFormula": "(!fb:finance.currency.countries_formerly_used fb:en.spanish_peseta)"},
{"utterance": "what is the currency code for the spanish peseta", "targetFormula": "(!fb:finance.currency.currency_code fb:en.spanish_peseta)"},
{"utterance": "how many awards did big daddy win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.big_daddy)))"},
{"utterance": "what bridges go over the san francisco bay", "targetFormula": "(!fb:geography.body_of_water.bridges fb:en.san_francisco_bay)"},
{"utterance": "how many notable people died by poisoning", "targetFormula": "(count (fb:people.deceased_person.cause_of_death fb:en.poison))"},
{"utterance": "what parks are in the canadian national parks system", "targetFormula": "(!fb:protected_sites.park_system.member_parks fb:en.canadian_national_parks)"},
{"utterance": "who instructed steven seagal", "targetFormula": "(!fb:martial_arts.martial_artist.instructor fb:en.steven_seagal)"},
{"utterance": "when did the last episode of six feet under air", "targetFormula": "(!fb:tv.tv_program.air_date_of_final_episode fb:en.six_feet_under)"},
{"utterance": "how many major events happened in australia", "targetFormula": "(count (!fb:location.location.events fb:en.australia))"},
{"utterance": "how many religions believe in reincarnation", "targetFormula": "(count (!fb:religion.belief.belief_of fb:en.reincarnation))"},
{"utterance": "what other titles does 13 going on 30 have", "targetFormula": "(!fb:common.topic.alias fb:en.13_going_on_30)"},
{"utterance": "how many writing systems are used in japanese", "targetFormula": "(count (!fb:language.human_language.writing_system fb:en.japanese))"},
{"utterance": "who is the ceo of savealot", "targetFormula": "(!fb:organization.leadership.person (and ((lambda x (fb:organization.leadership.role (var x))) fb:en.chief_executive_officer) ((lambda x (fb:organization.leadership.organization (var x))) fb:en.save-a-lot)))"},
{"utterance": "what starts are in ursa minor", "targetFormula": "(!fb:astronomy.constellation.contains fb:en.ursa_minor)"},
{"utterance": "how many conferences have been held at the los angeles convention center", "targetFormula": "(count (!fb:conferences.conference_venue.conferences fb:en.los_angeles_convention_center))"},
{"utterance": "when was diet mountain dew created", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.mountain_dew))"},
{"utterance": "what are some mountain bike models", "targetFormula": "(!fb:bicycles.bicycle_type.bicycle_models_of_this_type fb:en.mountain_bike)"},
{"utterance": "how many films has julie andrews been in", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.julie_edwards)))"},
{"utterance": "who designed the costumes for alice in wonderland", "targetFormula": "(!fb:film.film.costume_design_by fb:m.04jpg2p)"},
{"utterance": "what mountain range is king edward peak in", "targetFormula": "(!fb:geography.mountain.mountain_range fb:en.king_edward_peak)"},
{"utterance": "what are some hotels in vancouver", "targetFormula": "(!fb:travel.travel_destination.accommodation fb:en.vancouver_british_columbia)"},
{"utterance": "what year were the ny yankees founded", "targetFormula": "(!fb:sports.sports_team.founded fb:en.new_york_yankees)"},
{"utterance": "how many national parks does the national wildlife refuge have", "targetFormula": "(count (!fb:protected_sites.park_system.member_parks fb:en.national_wildlife_refuge))"},
{"utterance": "how many actors use their middle name or initial", "targetFormula": "(count (!fb:freebase.list.entries fb:m.0599h7b))"},
{"utterance": "where is fry_s turkish delight sold", "targetFormula": "(!fb:food.candy_bar.sold_in fb:en.frys_turkish_delight)"},
{"utterance": "who invented koolaid", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.kool-aid))"},
{"utterance": "when was doritos produced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.doritos))"},
{"utterance": "how many countries use the spanish peseta", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.spanish_peseta))"},
{"utterance": "when was the pencarrow head lighthouse first lit", "targetFormula": "(!fb:architecture.lighthouse.year_first_lit fb:en.pencarrow_head_lighthouse)"},
{"utterance": "what titles does the world boxing association have", "targetFormula": "(!fb:boxing.boxing_sanctioning_body.titles fb:en.world_boxing_association)"},
{"utterance": "how many species does the san francisco zoo have", "targetFormula": "(!fb:zoos.zoo.num_species fb:en.san_francisco_zoo)"},
{"utterance": "what round did thrilla in manila end", "targetFormula": "(!fb:boxing.boxing_match.round_match_ended fb:en.thrilla_in_manila)"},
{"utterance": "what is pycon about", "targetFormula": "(!fb:conferences.conference_series.subject fb:en.pycon)"},
{"utterance": "how many works have been lost due to theft", "targetFormula": "(count (!fb:media_common.cause_of_loss.works_lost_this_way fb:en.theft))"},
{"utterance": "what armed forces does thailand have", "targetFormula": "(!fb:military.military_combatant.armed_forces fb:en.thailand)"},
{"utterance": "what martial arts does chris maden practice", "targetFormula": "(!fb:martial_arts.martial_artist.martial_art fb:en.chris_maden)"},
{"utterance": "when was koolaid invented", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.kool-aid))"},
{"utterance": "what year did sgt. pepper's lonely hearts club band win a grammy", "targetFormula": "(!fb:award.award_honor.year ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.sgt_peppers_lonely_hearts_club_band))"},
{"utterance": "how many islands are there in lake superior", "targetFormula": "(count (!fb:geography.body_of_water.islands fb:en.lake_superior))"},
{"utterance": "how many countries is spanish spoken in", "targetFormula": "(count (!fb:language.human_language.countries_spoken_in fb:en.spanish))"},
{"utterance": "when was the ss great britain established as a museum", "targetFormula": "(!fb:architecture.museum.established fb:en.ss_great_britain)"},
{"utterance": "what type of rock is the marcellus formation", "targetFormula": "(!fb:geology.geological_formation.type_of_rock fb:en.marcellus_formation)"},
{"utterance": "when was capri sun developed", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.capri_sun))"},
{"utterance": "when was 300 released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.300_2007)"},
{"utterance": "how many wins did the philadelphia eagles have in the 2008 nfl season", "targetFormula": "(count (!fb:sports.sports_team_season_record.wins (and ((lambda x (fb:sports.sports_team_season_record.season (var x))) fb:en.2008_nfl_season) ((lambda x (fb:sports.sports_team_season_record.team (var x))) fb:en.philadelphia_eagles))))"},
{"utterance": "when was barbie launched", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.barbie))"},
{"utterance": "how many libretti did wagner write", "targetFormula": "(count (!fb:opera.librettist.libretti fb:en.richardwagner))"},
{"utterance": "what is bruce almighty rated", "targetFormula": "(!fb:film.film.rating fb:en.bruce_almighty)"},
{"utterance": "where was lady washington built", "targetFormula": "(!fb:boats.ship.place_built fb:en.lady_washington)"},
{"utterance": "how many domains are in the science & technology category", "targetFormula": "(count (!fb:freebase.domain_category.domains fb:en.science_technology))"},
{"utterance": "what is the building function of the eiffel tower", "targetFormula": "(!fb:architecture.building.building_function fb:en.eiffel_tower)"},
{"utterance": "how many children does danny devito have", "targetFormula": "(count (!fb:people.person.children fb:en.danny_devito))"},
{"utterance": "how many different industries are there in home depot", "targetFormula": "(count (!fb:business.business_operation.industry fb:en.the_home_depot))"},
{"utterance": "at what school was delta delta delta founded", "targetFormula": "(!fb:education.fraternity_sorority.founded_location fb:en.delta_delta_delta)"},
{"utterance": "in what season of stargate sg-1 is the episode show and tell", "targetFormula": "(!fb:tv.tv_series_episode.season fb:m.08mpny)"},
{"utterance": "who started starbucks", "targetFormula": "(!fb:organization.organization.founders fb:en.starbucks)"},
{"utterance": "who is the captain of the edmonton oilers", "targetFormula": "(!fb:ice_hockey.hockey_team.captain fb:en.edmonton_oilers)"},
{"utterance": "who are the founders of home depot", "targetFormula": "(!fb:organization.organization.founders fb:en.the_home_depot)"},
{"utterance": "what are the celtic languages", "targetFormula": "(!fb:language.language_family.languages fb:en.celtic_languages)"},
{"utterance": "what is the state flower of alaska", "targetFormula": "(!fb:location.location_symbol_relationship.symbol (and ((lambda x (fb:location.location_symbol_relationship.administrative_division (var x))) fb:en.alaska) ((lambda x (fb:location.location_symbol_relationship.Kind_of_symbol (var x))) fb:en.state_flower)))"},
{"utterance": "what are the theme areas at disneyland", "targetFormula": "(!fb:amusement_parks.park.areas fb:en.disneyland)"},
{"utterance": "what production is the sopranos currently in", "targetFormula": "(!fb:tv.tv_program.currently_in_production fb:en.the_sopranos)"},
{"utterance": "where does the chow chow originate", "targetFormula": "(!fb:biology.animal_breed.place_of_origin fb:en.chow_chow)"},
{"utterance": "what position did mike schmidt play", "targetFormula": "(!fb:baseball.baseball_player.position_s fb:en.mike_schmidt)"},
{"utterance": "what breed group is a shar pei in the american kennel club", "targetFormula": "(!fb:biology.breed_registration.breed_group (and ((lambda x (fb:biology.breed_registration.breed (var x))) fb:en.shar_pei) ((lambda x (fb:biology.breed_registration.registry (var x))) fb:en.american_kennel_club)))"},
{"utterance": "where was polonium discovered", "targetFormula": "(!fb:chemistry.chemical_element.discovering_country fb:en.polonium)"},
{"utterance": "how many episodes of taylor made piano were there", "targetFormula": "(count (!fb:radio.radio_program.episodes fb:en.taylor_made_piano_a_jazz_history))"},
{"utterance": "what record label was ali farka toure signed to", "targetFormula": "(!fb:music.artist.label fb:en.ali_farka_toure)"},
{"utterance": "what team does richard hamilton play for", "targetFormula": "(!fb:basketball.basketball_roster_position.team ((lambda x (fb:basketball.basketball_roster_position.player (var x))) fb:en.richard_hamilton))"},
{"utterance": "what is africa 's population", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.africa))"},
{"utterance": "what are the isotopes of zinc", "targetFormula": "(!fb:chemistry.chemical_element.isotopes fb:en.zinc)"},
{"utterance": "who published the amazing spider-man", "targetFormula": "(!fb:comic_books.comic_book_series.publisher fb:en.the_amazing_spider-man)"},
{"utterance": "how thick is the aletsch glacier", "targetFormula": "(!fb:geography.glacier.thickness fb:en.aletsch_glacier)"},
{"utterance": "when was the release date for titanic", "targetFormula": "(!fb:film.film.initial_release_date fb:en.titanic_special_edition_dvd)"},
{"utterance": "what are some research only cancer centers", "targetFormula": "(!fb:medicine.cancer_center_type.centers_of_this_kind fb:en.research_only)"},
{"utterance": "what is the theme song of full house", "targetFormula": "(!fb:tv.tv_program.theme_song fb:en.full_house)"},
{"utterance": "how was pluto discovered", "targetFormula": "(!fb:astronomy.astronomical_discovery.discovery_technique fb:en.pluto)"},
{"utterance": "what versions of mac os x is mozilla firefox compatible with", "targetFormula": "(!fb:computer.software_compatibility.operating_system (!fb:computer.software.compatible_oses fb:en.mozilla_firefox))"},
{"utterance": "how many seasons of seinfeld are there", "targetFormula": "(count (!fb:tv.tv_program.seasons fb:en.seinfeld))"},
{"utterance": "in 1982 who were the primetieme emmy award for comedy series nominees", "targetFormula": "(!fb:award.award_nomination.award_nominee (and ((lambda x (fb:award.award_nomination.year (var x))) (date 1982 -1 -1)) ( (lambda x (fb:award.award_nomination.award (var x))) fb:en.primetime_emmy_award_for_outstanding_comedy_series)))"},
{"utterance": "what company owns nutter butter", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.nutter_butter))"},
{"utterance": "when was facebook launched", "targetFormula": "(!fb:internet.website.launched fb:en.facebook)"},
{"utterance": "what position does cristiano ronaldo play", "targetFormula": "(!fb:soccer.football_player.position_s fb:en.cristiano_ronaldo)"},
{"utterance": "how many games has hasbro published", "targetFormula": "(count (!fb:games.game_publisher.games_published fb:en.hasbro))"},
{"utterance": "what universe is the lord of the rings set in", "targetFormula": "(!fb:fictional_universe.work_of_fiction.setting fb:en.the_lord_of_the_rings)"},
{"utterance": "how many teams participated in the 1979 cricket world cup", "targetFormula": "(count (!fb:cricket.cricket_tournament_event.teams fb:en.1979_cricket_world_cup))"},
{"utterance": "what year was the album decade released", "targetFormula": "(!fb:music.album.release_date fb:m.02967)"},
{"utterance": "how many operating systems is adobe flash compatible with", "targetFormula": "(count (!fb:computer.software_compatibility.operating_system ((lambda x (fb:computer.software_compatibility.software (var x))) fb:en.adobe_flash)))"},
{"utterance": "what is the population estimated in the world", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.earth))"},
{"utterance": "how many tennis events are there at the olympics", "targetFormula": "(count (!fb:olympics.olympic_sport.events fb:m.07bs0))"},
{"utterance": "how many rna codons does glycine have", "targetFormula": "(count (!fb:biology.amino_acid.rna_codons fb:en.glycine))"},
{"utterance": "who was charlie_s angels produced by", "targetFormula": "(!fb:film.film.produced_by fb:en.charlies_angels)"},
{"utterance": "who completed mozart_s requiem", "targetFormula": "(!fb:media_common.completion_of_unfinished_work.finisher ((lambda x (fb:media_common.completion_of_unfinished_work.unfinished_work (var x))) fb:m.02xtrr))"},
{"utterance": "how many teams did joe torre manage", "targetFormula": "(count (!fb:baseball.baseball_historical_managerial_position.team ((lambda x (fb:baseball.baseball_historical_managerial_position.manager (var x))) fb:en.joe_torre)))"},
{"utterance": "how many cricket bowlers use fast bowling", "targetFormula": "(count (fb:cricket.cricket_bowler.technique fb:en.fast_bowling))"},
{"utterance": "who designed the giant dipper", "targetFormula": "(!fb:amusement_parks.ride.designer fb:en.giant_dipper)"},
{"utterance": "what characters are in super mario bros", "targetFormula": "(!fb:cvg.game_performance.character ((lambda x (fb:cvg.game_performance.game (var x))) fb:en.super_mario_bros))"},
{"utterance": "what was john f kennedy 's cause of death", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.john_f_kennedy)"},
{"utterance": "what is the genre of the skeptics' guide to the universe", "targetFormula": "(!fb:broadcast.content.genre fb:m.03z9smt)"},
{"utterance": "how many employees does nintendo have", "targetFormula": "(count (!fb:business.employment_tenure.person (!fb:business.employer.employees fb:en.nintendo)))"},
{"utterance": "how many types of cumulus clouds are there", "targetFormula": "(count (!fb:meteorology.cloud.varieties fb:m.0csh5))"},
{"utterance": "how many other names is ron glass known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.ron_glass))"},
{"utterance": "what number is ryan callahan on the new york rangers", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.number (and ((lambda x (fb:ice_hockey.hockey_roster_position.team (var x))) fb:en.new_york_rangers) ((lambda x (fb:ice_hockey.hockey_roster_position.player (var x))) fb:en.ryan_callahan)))"},
{"utterance": "what are the christian holidays", "targetFormula": "(!fb:religion.religion.holidays fb:en.christianity)"},
{"utterance": "who directed charlie_s angels", "targetFormula": "(!fb:film.film.directed_by fb:en.charlies_angels)"},
{"utterance": "what was jack albertson 's cause of death", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.jack_albertson)"},
{"utterance": "when was the printing press invented", "targetFormula": "(!fb:law.invention.date_of_invention fb:en.printing_press)"},
{"utterance": "what are the neighborhoods in new york city", "targetFormula": "(!fb:location.place_with_neighborhoods.neighborhoods fb:en.new_york_ny)"},
{"utterance": "how many people practice karate", "targetFormula": "(count (!fb:martial_arts.martial_art.well_known_practitioner fb:en.karate))"},
{"utterance": "what is the area of south america", "targetFormula": "(!fb:location.location.area fb:en.south_america)"},
{"utterance": "how many sites are on the national register of historic places", "targetFormula": "(count (!fb:protected_sites.natural_or_cultural_site_listing.listed_site ( (lambda x (fb:protected_sites.natural_or_cultural_site_listing.designation (var x))) fb:en.national_register_of_historic_places)))"},
{"utterance": "what football team does andy reid currently coach", "targetFormula": "(!fb:american_football.football_coach.current_team_head_coached fb:en.andy_reid)"},
{"utterance": "how many protected sites does the u.s. national park service govern", "targetFormula": "(count (!fb:protected_sites.governing_body_of_protected_sites.protected_sites_governed fb:en.national_park_service))"},
{"utterance": "what was viacom 's revenue in 2009", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (and ((lambda x (fb:measurement_unit.dated_money_value.valid_date (var x))) (date 2009 -1 -1)) ((lambda x (!fb:business.business_operation.revenue (var x))) fb:en.viacom)))"},
{"utterance": "what island group is jekyll island a part of", "targetFormula": "(!fb:geography.island.island_group fb:en.jekyll_island)"},
{"utterance": "how many children does jerry seinfeld have", "targetFormula": "(count (!fb:people.person.children fb:en.jerry_seinfeld))"},
{"utterance": "what type of organism was lucy a fossil of", "targetFormula": "(!fb:biology.fossil_specimen.organism fb:m.0gd2g7)"},
{"utterance": "when was crystal light originally marketed", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.crystal_light))"},
{"utterance": "how many people ride the london underground daily", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:metropolitan_transit.transit_system.daily_riders fb:en.london_underground)))"},
{"utterance": "in what disaster was old st paul_s cathedral destroyed", "targetFormula": "(!fb:event.disaster_affected_structure.destroyed_by_disaster fb:en.old_st_pauls_cathedral)"},
{"utterance": "what instruments did omarion play", "targetFormula": "(!fb:music.group_member.instruments_played fb:en.omarion_grandberry)"},
{"utterance": "what are the books in the chronicles of narnia series", "targetFormula": "(fb:book.written_work.part_of_series fb:en.the_chronicles_of_narnia)"},
{"utterance": "what characters are featured in batman: the dark knight returns", "targetFormula": "(!fb:comic_books.comic_book_series.featured_characters fb:en.batman_the_dark_knight_returns)"},
{"utterance": "what was henry viii's royal line", "targetFormula": "(!fb:royalty.monarch.royal_line fb:en.king_henry_viii_of_england)"},
{"utterance": "what was the cost of building the magnum xl-200", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:amusement_parks.ride.cost fb:en.magnum_xl-200))"},
{"utterance": "how many cow's milk cheeses are there", "targetFormula": "(count (!fb:food.cheese_milk_source.cheeses fb:en.cattle))"},
{"utterance": "did the big bang exhibit at the science museum cost money", "targetFormula": "(!fb:exhibitions.exhibition_run.admission_fee (and ((lambda x (fb:exhibitions.exhibition_run.venue (var x))) fb:en.science_museum_great_britain) ((lambda x (fb:exhibitions.exhibition_run.exhibition (var x))) fb:m.046chwc)))"},
{"utterance": "what movie did danny devito win an award for in 1981", "targetFormula": "(!fb:award.award_honor.honored_for (and ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.danny_devito) ((lambda x (fb:award.award_honor.year (var x))) (date 1981 -1 -1))))"},
{"utterance": "who coaches the australian cricket team", "targetFormula": "(!fb:cricket.cricket_team.coach fb:en.australian_cricket_team)"},
{"utterance": "what are the texts of taoism", "targetFormula": "(!fb:religion.religion.texts fb:en.taoism)"},
{"utterance": "who was 8 mile directed by", "targetFormula": "(!fb:film.film.directed_by fb:en.8_mile)"},
{"utterance": "how many tv did jerry seinfeld have a starring role in", "targetFormula": "(count (!fb:tv.regular_tv_appearance.series ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.jerry_seinfeld)))"},
{"utterance": "when was walmart founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.wal-mart)"},
{"utterance": "who used to be quarterback for the green bay packers", "targetFormula": "(!fb:american_football.football_historical_roster_position.player (and ( (lambda x (fb:american_football.football_historical_roster_position.position_s (var x))) fb:en.quarterback) ( (lambda x (fb:american_football.football_historical_roster_position.team (var x))) fb:en.green_bay_packers)))"},
{"utterance": "how many teams are in the atlantic division of the eastern conference", "targetFormula": "(count (!fb:ice_hockey.hockey_division.teams fb:en.atlantic_division))"},
{"utterance": "how many radio programs about science are there", "targetFormula": "(count (!fb:radio.radio_subject.programs_with_this_subject fb:m.06mq7))"},
{"utterance": "when was the latest release of microsoft word", "targetFormula": "(!fb:computer.software.latest_release_date fb:en.microsoft_word)"},
{"utterance": "what architectural style is the brooklyn bridge", "targetFormula": "(!fb:architecture.structure.architectural_style fb:en.brooklyn_bridge)"},
{"utterance": "how many tv channels does nbc own", "targetFormula": "(count (!fb:broadcast.tv_station_owner.tv_stations fb:en.nbc))"},
{"utterance": "how many turbojet engines are there", "targetFormula": "(count (!fb:engineering.engine_category.engines fb:en.turbojet))"},
{"utterance": "how many inversions does the mind eraser have", "targetFormula": "(count (!fb:amusement_parks.ride.inversions fb:en.the_mind_eraser))"},
{"utterance": "when was 13 going on 30 released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.13_going_on_30)"},
{"utterance": "when did john j. raskob own the empire state building", "targetFormula": "(!fb:architecture.ownership.end_date (and ((lambda x (fb:architecture.ownership.owner (var x))) fb:en.john_j_raskob) ((lambda x (fb:architecture.ownership.structure (var x))) fb:en.empire_state_building)))"},
{"utterance": "what was the date of the first sesame street episode aired", "targetFormula": "(!fb:tv.tv_program.air_date_of_first_episode fb:en.sesame_street)"},
{"utterance": "who created the far side", "targetFormula": "(!fb:comic_strips.comic_strip_creator_duration.creator_of_strip ((lambda x (fb:comic_strips.comic_strip_creator_duration.comic_strip (var x))) fb:en.the_far_side))"},
{"utterance": "what conditions have symptoms including headache", "targetFormula": "(fb:medicine.disease.symptoms fb:en.severe_headache)"},
{"utterance": "what games has macsoft games developed", "targetFormula": "(!fb:cvg.cvg_developer.games_developed fb:en.macsoft_games)"},
{"utterance": "are lithium batteries rechargable", "targetFormula": "(!fb:engineering.battery_cell_type.rechargeable fb:en.lithium_battery)"},
{"utterance": "how many people died of a skiing accident", "targetFormula": "(count (fb:people.deceased_person.cause_of_death fb:en.skiing_accident))"},
{"utterance": "what decision did manny pacquiao vs. timothy bradley end with", "targetFormula": "(!fb:boxing.boxing_match.decision fb:m.0j24b7h)"},
{"utterance": "what spirits are produced in kentucky", "targetFormula": "(!fb:distilled_spirits.spirit_producing_region.distilleries fb:en.kentucky)"},
{"utterance": "when was home depot founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.the_home_depot)"},
{"utterance": "when did japan end as a musical group", "targetFormula": "(!fb:music.artist.active_end fb:m.016f2t)"},
{"utterance": "what animal does marscapone cheese come from", "targetFormula": "(!fb:food.cheese.source_of_milk fb:en.mascarpone)"},
{"utterance": "who was titanic directed by", "targetFormula": "(!fb:film.film.directed_by fb:en.titanic_special_edition_dvd)"},
{"utterance": "what matches have had the wbc world champion title at stake", "targetFormula": "(!fb:boxing.boxing_title.matches_with_this_title_at_stake fb:m.0cj52b7)"},
{"utterance": "how many people played in the 2010 fifa world cup final", "targetFormula": "(count (!fb:soccer.football_player_match_participation.player ( (lambda x (fb:soccer.football_player_match_participation.match (var x))) fb:en.2010_fifa_world_cup_final)))"},
{"utterance": "how many celebrities have abused cocaine", "targetFormula": "(count (!fb:celebrities.substance_abuse_problem.celebrity ((lambda x (fb:celebrities.substance_abuse_problem.substance (var x))) fb:en.cocaine)))"},
{"utterance": "what are the lines of the new york city subway", "targetFormula": "(!fb:metropolitan_transit.transit_system.transit_lines fb:en.new_york_city_subway)"},
{"utterance": "when was cap'n crunch introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.capn_crunch))"},
{"utterance": "when was oxygen discovered", "targetFormula": "(!fb:chemistry.chemical_element.discovery_date fb:en.oxygen)"},
{"utterance": "when did easy aces stop being produced", "targetFormula": "(!fb:broadcast.content.production_end fb:en.easy_aces)"},
{"utterance": "what team does mike babcock coach", "targetFormula": "(!fb:ice_hockey.hockey_coach.current_team fb:en.mike_babcock)"},
{"utterance": "how many engineers worked on the design and construction of the plymouth breakwater", "targetFormula": "(count (!fb:projects.project_participation.participant (and ( (lambda x (fb:projects.project_participation.project (var x))) fb:en.design_and_construction_of_the_plymouth_breakwater) ((lambda x (fb:projects.project_participation.role (var x))) fb:en.engineer))))"},
{"utterance": "when was the sony nex-5 released", "targetFormula": "(!fb:digicams.digital_camera.released fb:m.0cp1pcv)"},
{"utterance": "when was letter from america last broadcast", "targetFormula": "(!fb:radio.radio_program.final_broadcast fb:en.letter_from_america)"},
{"utterance": "in what zoo was knut kept", "targetFormula": "(!fb:zoos.animal_captivity.zoo ((lambda x (fb:zoos.animal_captivity.animal (var x))) fb:m.02q470c))"},
{"utterance": "what is the highest drop on stealth", "targetFormula": "(!fb:amusement_parks.ride.drop fb:m.0b6_sx)"},
{"utterance": "how many films has tim burton produced", "targetFormula": "(count (fb:film.film.produced_by fb:en.tim_burton))"},
{"utterance": "did jack dempsey win the long count fight", "targetFormula": "(!fb:boxing.match_boxer_relationship.winner_won ((lambda x (fb:boxing.match_boxer_relationship.boxer (var x))) fb:en.jack_dempsey))"},
{"utterance": "who were the key designers of the macintosh", "targetFormula": "(!fb:computer.computer.key_designers fb:en.macintosh)"},
{"utterance": "how many people have won the nobel peace prize", "targetFormula": "(count (!fb:award.award_honor.award_winner ((lambda x (fb:award.award_honor.award (var x))) fb:en.nobel_peace_prize)))"},
{"utterance": "where was omarion born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.omarion_grandberry)"},
{"utterance": "what tourist attractions are in rome", "targetFormula": "(!fb:travel.travel_destination.tourist_attractions fb:en.rome)"},
{"utterance": "what are the major exports of madagascar", "targetFormula": "(!fb:location.imports_exports_by_industry.industry ((lambda x (!fb:location.statistical_region.major_exports (var x))) fb:en.madagascar))"},
{"utterance": "when was the construction of new steubenville bridge finished", "targetFormula": "(!fb:projects.project.actual_completion_date fb:m.0jb4_0h)"},
{"utterance": "how many other names are there for jcpenney", "targetFormula": "(count (!fb:common.topic.alias fb:en.j_c_penney))"},
{"utterance": "what language family is afrikaans part of", "targetFormula": "(!fb:language.human_language.language_family fb:en.afrikaans)"},
{"utterance": "what percentage of the grapes in a 1966 chateau latour grand vin are merlot", "targetFormula": "(!fb:wine.grape_variety_composition.percentage (and ((lambda x (fb:wine.grape_variety_composition.wine (var x))) fb:en.1966_chateau_latour) ((lambda x (fb:wine.grape_variety_composition.grape_variety (var x))) fb:en.merlot)))"},
{"utterance": "how many historical events happened in south america", "targetFormula": "(count (!fb:location.location.events fb:en.south_america))"},
{"utterance": "what meteor showers has the comet halley spawned", "targetFormula": "(!fb:astronomy.comet.meteor_shower_spawned fb:en.comet_halley)"},
{"utterance": "when was pride and prejudice published", "targetFormula": "(!fb:book.written_work.date_of_first_publication fb:en.pride_and_prejudice)"},
{"utterance": "who manufactured millennium force", "targetFormula": "(!fb:amusement_parks.ride.manufacturer fb:en.millennium_force)"},
{"utterance": "what characters were on the cover of batman #1", "targetFormula": "(!fb:comic_books.comic_book_issue.characters_on_cover fb:en.batman_1)"},
{"utterance": "what sort of weave is used to make tweed", "targetFormula": "(!fb:fashion.textile.weave fb:en.tweed)"},
{"utterance": "when gatorade was first developed", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.gatorade))"},
{"utterance": "on how many projects was james walker a design engineer", "targetFormula": "(count (!fb:projects.project_participation.project (and ((lambda x (fb:projects.project_participation.participant (var x))) fb:en.james_walker) ((lambda x (fb:projects.project_participation.role (var x))) fb:en.design_engineer))))"},
{"utterance": "how many awards did the movie 8 mile win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.8_mile)))"},
{"utterance": "how many people practice buddhism", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:religion.religion.number_of_adherents fb:en.buddhism))"},
{"utterance": "how much did the construction of the taj mahal cost", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:projects.project.actual_cost fb:en.initial_design_and_construction_of_taj_mahal))"},
{"utterance": "what are some object-oriented programming languages", "targetFormula": "(!fb:computer.programming_language_paradigm.languages fb:en.object-oriented_programming)"},
{"utterance": "who were the curators for renoir in the 20th century", "targetFormula": "(!fb:exhibitions.exhibition.curators fb:en.renoir_in_the_20th_century)"},
{"utterance": "for what country did bernard lagat play in the 2000 summer olympics", "targetFormula": "(!fb:olympics.olympic_athlete_affiliation.country (and ((lambda x (fb:olympics.olympic_athlete_affiliation.olympics (var x))) fb:en.2000_summer_olympics) ((lambda x (fb:olympics.olympic_athlete_affiliation.athlete (var x))) fb:en.bernard_lagat)))"},
{"utterance": "what titles were at stake in the the rumble in the jungle", "targetFormula": "(!fb:boxing.boxing_match.titles_at_stake fb:en.the_rumble_in_the_jungle)"},
{"utterance": "what games has electronic arts developed", "targetFormula": "(!fb:cvg.cvg_developer.games_developed fb:en.electronic_arts)"},
{"utterance": "how many stores are in nittany mall", "targetFormula": "(count (!fb:business.shopping_center.store fb:en.nittany_mall))"},
{"utterance": "when was nutty professor released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.the_nutty_professor_1996)"},
{"utterance": "who was the editor in chief of die welt in 2000", "targetFormula": "(!fb:business.employment_tenure.person (and (and ((lambda x (fb:business.employment_tenure.company (var x))) fb:en.die_welt) ((lambda x (fb:business.employment_tenure.to (var x))) (date 2000 -1 -1))) ((lambda x (fb:business.employment_tenure.title (var x))) fb:en.editor_in_chief)))"},
{"utterance": "what is the nutty professor rated", "targetFormula": "(!fb:film.film.rating fb:en.the_nutty_professor_1996)"},
{"utterance": "how many film performances did jack albertson do", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.jack_albertson)))"},
{"utterance": "what musicians have died of lung cancer", "targetFormula": "(fb:people.deceased_person.cause_of_death fb:en.lung_cancer)"},
{"utterance": "what type of bridge is the manhattan bridge", "targetFormula": "(!fb:transportation.bridge.bridge_type fb:en.manhattan_bridge)"},
{"utterance": "what sport did scott anderson play in the 1992 summer olympics", "targetFormula": "(!fb:olympics.olympic_athlete_affiliation.sport (and ((lambda x (fb:olympics.olympic_athlete_affiliation.athlete (var x))) fb:m.04dnjr9) ((lambda x (fb:olympics.olympic_athlete_affiliation.olympics (var x))) fb:en.1992_summer_olympics)))"},
{"utterance": "who designed the parthenon", "targetFormula": "(!fb:architecture.structure.architect fb:en.parthenon)"},
{"utterance": "how many monarchs are from the house of tutor", "targetFormula": "(count (!fb:royalty.royal_line.monarchs_from_this_line fb:en.tudor_dynasty))"},
{"utterance": "what german athletes have participated in the olympics", "targetFormula": "(!fb:olympics.olympic_athlete_affiliation.athlete ((lambda x (fb:olympics.olympic_athlete_affiliation.country (var x))) fb:en.germany))"},
{"utterance": "what teams did babe ruth play for", "targetFormula": "(!fb:baseball.baseball_historical_roster_position.team ((lambda x (fb:baseball.baseball_historical_roster_position.player (var x))) fb:en.babe_ruth))"},
{"utterance": "in what events did ian thorpe compete in the 2004 summer olympics", "targetFormula": "(!fb:olympics.olympic_medal_honor.event (and ((lambda x (fb:olympics.olympic_medal_honor.olympics (var x))) fb:en.2004_summer_olympics) ((lambda x (fb:olympics.olympic_medal_honor.medalist (var x))) fb:en.ian_thorpe)))"},
{"utterance": "what is the wingspan of an eclipse 500", "targetFormula": "(!fb:aviation.aircraft_model.wingspan fb:en.eclipse_500)"},
{"utterance": "who sponsors the hoby seminars", "targetFormula": "(!fb:conferences.conference_series.sponsoring_organization fb:m.0hmwcy2)"},
{"utterance": "who founded the order of the dragon", "targetFormula": "(!fb:royalty.order_of_chivalry.founders fb:en.order_of_the_dragon)"},
{"utterance": "who produced sabotage by the beastie boys", "targetFormula": "(!fb:music.recording.producer fb:m.0l16j8)"},
{"utterance": "how many runs does the thunder ridge ski area have", "targetFormula": "(count (!fb:skiing.ski_area.number_of_runs fb:en.thunder_ridge_ski_area))"},
{"utterance": "how many people died in hurricane wilma", "targetFormula": "(!fb:meteorology.tropical_cyclone.direct_fatalities fb:en.hurricane_wilma)"},
{"utterance": "who discovered the rings of saturn", "targetFormula": "(!fb:astronomy.astronomical_discovery.discoverer fb:en.rings_of_saturn)"},
{"utterance": "what is the lcd screen resolution of a nikon d80", "targetFormula": "(!fb:digicams.digital_camera.lcd_pixels fb:en.nikon_d80)"},
{"utterance": "when was the san diego zoo opened", "targetFormula": "(!fb:zoos.zoo.opened fb:en.san_diego_zoo)"},
{"utterance": "how heavy is a panasonic lumix dmc-tz3", "targetFormula": "(!fb:digicams.digital_camera.weight fb:en.panasonic_lumix_dmc_tz3)"},
{"utterance": "where was liam gallagher born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.liam_gallagher)"},
{"utterance": "how many beers come a can", "targetFormula": "(count (!fb:food.beer_containment.beer ((lambda x (fb:food.beer_containment.container_type (var x))) fb:en.beverage_can)))"},
{"utterance": "what is the subject of the atlantic monthly", "targetFormula": "(!fb:book.periodical.subjects fb:en.the_atlantic_monthly)"},
{"utterance": "what was the american past about", "targetFormula": "(!fb:radio.radio_program.subjects fb:m.05v0tbd)"},
{"utterance": "who destroyed the one ring", "targetFormula": "(!fb:fictional_universe.fictional_object.destroyed_by fb:en.one_ring)"},
{"utterance": "how many students are there at the university of iceland", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:education.educational_institution.total_enrollment fb:en.university_of_iceland))"},
{"utterance": "who founded the red cross", "targetFormula": "(!fb:organization.organization.founders fb:en.american_red_cross)"},
{"utterance": "how many radio stations does cbs radio own", "targetFormula": "(count (!fb:broadcast.radio_station_owner.radio_stations fb:en.cbs_radio))"},
{"utterance": "what is ashok malhotra's bowling pace", "targetFormula": "(!fb:cricket.cricket_bowler.pace fb:en.ashok_malhotra)"},
{"utterance": "how long is wired_s gadget lab podcast", "targetFormula": "(!fb:broadcast.podcast_feed.average_media_length fb:en.wireds_gadget_lab_podcast_podcast_feed)"},
{"utterance": "how many politicians have served in the us navy", "targetFormula": "(count (!fb:military.military_service.military_person (!fb:military.armed_force.personnel fb:en.united_states_navy)))"},
{"utterance": "what causes bipolar disorder", "targetFormula": "(!fb:medicine.disease.causes fb:en.bipolar_disorder)"},
{"utterance": "what causes syphilis", "targetFormula": "(!fb:medicine.disease.causes fb:en.syphillis)"},
{"utterance": "who said that_s one small step for man, one giant leap for mankind", "targetFormula": "(!fb:media_common.quotation.author fb:m.06gj5j3)"},
{"utterance": "what bicycle models does raleigh manufacture", "targetFormula": "(!fb:bicycles.bicycle_manufacturer.bicycle_models fb:en.raleigh_bicycle_company)"},
{"utterance": "what play was west side story adapted from", "targetFormula": "(!fb:media_common.adaptation.adapted_from fb:en.west_side_story)"},
{"utterance": "what team does alan butcher coach", "targetFormula": "(!fb:cricket.cricket_coach.current_team fb:en.alan_butcher)"},
{"utterance": "how many speeches have been given about world war ii", "targetFormula": "(count (!fb:event.speech_or_presentation.presented_work ((lambda x (fb:event.speech_or_presentation.speech_topic (var x))) fb:en.world_war_ii)))"},
{"utterance": "who is the newscaster on abc 6 news", "targetFormula": "(!fb:business.employment_tenure.person (and ((lambda x (fb:business.employment_tenure.title (var x))) fb:en.news_presenter) ((lambda x (fb:business.employment_tenure.company (var x))) fb:en.abc_news_washington_dc)))"},
{"utterance": "who is the present newscaster on cbs evening news", "targetFormula": "(!fb:tv.tv_regular_personal_appearance.person (and ((lambda x (fb:tv.tv_regular_personal_appearance.appearance_type (var x))) fb:en.newscaster) ((lambda x (fb:tv.tv_regular_personal_appearance.program (var x))) fb:en.cbs_evening_news)))"}
]

643
freebase/data/free917.train.examples.canonicalized.json Normal file
View File

@ -0,0 +1,643 @@
[
{"utterance": "at what institutions was marshall hall a professor", "targetFormula": "(!fb:education.academic_post.institution (and ((lambda x (fb:education.academic_post.person (var x))) fb:en.marshall_hall) ((lambda x (fb:education.academic_post.position_or_title (var x))) fb:en.professor)))"},
{"utterance": "what fuel does an internal combustion engine use", "targetFormula": "(!fb:engineering.engine.energy_source fb:en.internal_combustion_engine)"},
{"utterance": "how many companies are traded by the nyse", "targetFormula": "(count (!fb:business.stock_ticker_symbol.ticker_symbol (!fb:finance.stock_exchange.companies_traded fb:en.new_york_stock_exchange_inc)))"},
{"utterance": "what is the address of the apple, inc. headquarters", "targetFormula": "(!fb:location.mailing_address.citytown ((lambda x (!fb:organization.organization.headquarters (var x))) fb:en.apple_inc))"},
{"utterance": "what was the cover price of the x-men issue 1", "targetFormula": "(!fb:measurement_unit.money_value.amount (!fb:comic_books.comic_book_issue.cover_price fb:en.the_x_men_1))"},
{"utterance": "how often does the washington post come out", "targetFormula": "(!fb:book.periodical_frequency.issues_per_year (!fb:book.periodical.frequency_or_issues_per_year fb:en.the_washington_post))"},
{"utterance": "what issue of sandman is a dream of a thousand cats", "targetFormula": "(!fb:comic_books.comic_book_issue.issue_number fb:en.a_dream_of_a_thousand_cats)"},
{"utterance": "how many people survived the sinking of the titanic", "targetFormula": "(count (!fb:event.disaster.survivors fb:en.sinking_of_the_titanic))"},
{"utterance": "what olympics has egypt participated in", "targetFormula": "(!fb:olympics.olympic_participating_country.olympics_participated_in fb:en.egypt)"},
{"utterance": "what celebrities have gone to the betty ford center", "targetFormula": "(!fb:celebrities.rehab.celebrity ((lambda x (fb:celebrities.rehab.rehab_facility (var x))) fb:en.betty_ford_center))"},
{"utterance": "how old do you have to be to play monopoly", "targetFormula": "(!fb:games.game.minimum_age_years fb:en.monopoly_boardgame)"},
{"utterance": "how many first generation particles are there", "targetFormula": "(count (!fb:physics.subatomic_particle_generation.particles fb:m.0b66f5g))"},
{"utterance": "when was the first amendment ratified", "targetFormula": "(!fb:law.constitutional_amendment.ratification_completed_on fb:en.first_amendment_to_the_united_states_constitution)"},
{"utterance": "how many teams participate in the uefa", "targetFormula": "(count (!fb:sports.sports_league_participation.team ((lambda x (fb:sports.sports_league_participation.league (var x))) fb:en.uefa)))"},
{"utterance": "who are the linebackers on the chicago bears", "targetFormula": "(!fb:american_football.football_roster_position.player (and ((lambda x (fb:american_football.football_roster_position.position (var x))) fb:en.linebacker) ((lambda x (fb:american_football.football_roster_position.team (var x))) fb:en.chicago_bears)))"},
{"utterance": "what did bertrand russell die of", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.bertrand_russell)"},
{"utterance": "how long is the haunted mansion ride at disneyland", "targetFormula": "(!fb:amusement_parks.ride.duration fb:en.haunted_mansion)"},
{"utterance": "what type of tea is gunpowder tea", "targetFormula": "(!fb:food.tea.tea_type fb:en.gunpowder_tea)"},
{"utterance": "what highway system does us route 101 belong to", "targetFormula": "(!fb:transportation.road.highway_system fb:en.us_route_101)"},
{"utterance": "what league is real madrid a member of", "targetFormula": "(!fb:soccer.football_league_participation.league ((lambda x (fb:soccer.football_league_participation.team (var x))) fb:en.real_madrid))"},
{"utterance": "how much did a canon powershot tx1 cost when it was released", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:digicams.digital_camera.street_price fb:en.powershot_tx1))"},
{"utterance": "what regions does the lse cover", "targetFormula": "(!fb:finance.stock_exchange.primary_regions fb:en.london_stock_exchange)"},
{"utterance": "how many people does borders employ", "targetFormula": "(count (!fb:business.employment_tenure.person (!fb:business.employer.employees fb:en.borders_group)))"},
{"utterance": "where did monsanto rank on the forune 500 in 2000", "targetFormula": "(!fb:award.ranking.rank (and (and ((lambda x (fb:award.ranking.year (var x))) (date 2000 -1 -1)) ((lambda x (fb:award.ranking.list (var x))) fb:en.fortune_500)) ((lambda x (fb:award.ranking.item (var x))) fb:en.monsanto)))"},
{"utterance": "how many prosthetic makeup artists worked on 28 days later", "targetFormula": "(count (!fb:film.film_crew_gig.crewmember (and ((lambda x (fb:film.film_crew_gig.film_crew_role (var x))) fb:en.prosthetic_makeup_artist) ((lambda x (fb:film.film_crew_gig.film (var x))) fb:en.28_weeks_later))))"},
{"utterance": "how many ships has nathanael herreshoff designed", "targetFormula": "(count (!fb:boats.ship_designer.boats_designed fb:en.nathanael_herreshoff))"},
{"utterance": "who manages fulham f. c.", "targetFormula": "(!fb:soccer.football_team_management_tenure.manager ((lambda x (fb:soccer.football_team_management_tenure.team (var x))) fb:en.fulham_fc))"},
{"utterance": "when was the catcher in the rye first published", "targetFormula": "(!fb:book.written_work.date_of_first_publication fb:en.the_catcher_in_the_rye)"},
{"utterance": "how many rock formations were formed during the cretaceous period", "targetFormula": "(count (fb:geology.geological_formation.formed_during_period fb:en.cretaceous))"},
{"utterance": "what are bmw's manufacturing plants", "targetFormula": "(!fb:automotive.company.manufacturing_plants fb:en.bmw)"},
{"utterance": "who are some practitioners of judo", "targetFormula": "(!fb:martial_arts.martial_art.well_known_practitioner fb:en.judo)"},
{"utterance": "what is the average temperature in sydney in august", "targetFormula": "(!fb:travel.travel_destination_monthly_climate.average_max_temp_c (and ((lambda x (fb:travel.travel_destination_monthly_climate.travel_destination (var x))) fb:en.sydney) ((lambda x (fb:travel.travel_destination_monthly_climate.month (var x))) fb:en.august)))"},
{"utterance": "over what area does the court of chancery have jurisdiction", "targetFormula": "(!fb:law.court.jurisdiction fb:en.court_of_chancery)"},
{"utterance": "when did mount fuji last erupt", "targetFormula": "(!fb:geography.mountain.last_eruption fb:en.mount_fuji)"},
{"utterance": "how tall is westminster abbey", "targetFormula": "(!fb:architecture.structure.height_meters fb:en.westminster_abbey)"},
{"utterance": "when was wells fargo founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.wells_fargo)"},
{"utterance": "when did hey arnold! stop running", "targetFormula": "(!fb:tv.tv_program.air_date_of_final_episode fb:en.hey_arnold)"},
{"utterance": "when did christopher higgins stop playing for the montreal canadiens", "targetFormula": "(!fb:ice_hockey.hockey_previous_roster_position.to (and ( (lambda x (fb:ice_hockey.hockey_previous_roster_position.player (var x))) fb:en.christopher_higgins) ((lambda x (fb:ice_hockey.hockey_previous_roster_position.team (var x))) fb:en.montreal_canadiens)))"},
{"utterance": "what is the hull made of on the james craig", "targetFormula": "(!fb:boats.ship.hull_material fb:m.02zl93)"},
{"utterance": "how many pets did john f kennedy own", "targetFormula": "(count (!fb:base.famouspets.pet_ownership.pet ((lambda x (fb:base.famouspets.pet_ownership.owner (var x))) fb:en.john_f_kennedy)))"},
{"utterance": "how many floors does the white house have", "targetFormula": "(!fb:architecture.building.floors fb:en.white_house)"},
{"utterance": "in which comic book issue did kitty pryde first appear", "targetFormula": "(!fb:comic_books.comic_book_character.first_appearance fb:en.kitty_pryde)"},
{"utterance": "what are the deities of hinduism", "targetFormula": "(!fb:religion.religion.deities fb:en.hinduism)"},
{"utterance": "who designed the iphone", "targetFormula": "(!fb:computer.computer.key_designers fb:en.iphone)"},
{"utterance": "how many other names is jerry seinfeld known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.jerry_seinfeld))"},
{"utterance": "what are the latitude and longitude of the eiffel tower", "targetFormula": "(!fb:location.geocode.longitude (!fb:location.location.geolocation fb:en.eiffel_tower))"},
{"utterance": "how many visits does the magic kingdom get per year", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:amusement_parks.park.annual_visits fb:en.magic_kingdom))"},
{"utterance": "how many film performances did beyonce do", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.beyonce)))"},
{"utterance": "who manufactures the specialized stumpjumper", "targetFormula": "(!fb:bicycles.bicycle_model.manufacturer fb:en.stumpjumper)"},
{"utterance": "what types of fuel does a 4 cylinder engine take", "targetFormula": "(!fb:automotive.engine.fuels_used fb:en.4_cylinder)"},
{"utterance": "what major events in history happened in africa", "targetFormula": "(!fb:location.location.events fb:en.africa)"},
{"utterance": "what is the transmission of a 2011 honda fit", "targetFormula": "(!fb:automotive.trim_level.transmission fb:m.0h2krn6)"},
{"utterance": "what is angelina jolie's net worth", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount (!fb:celebrities.celebrity.net_worth fb:en.angelina_jolie))"},
{"utterance": "what conferences does google sponsor", "targetFormula": "(!fb:conferences.conference_sponsor.conferences fb:en.google)"},
{"utterance": "what is the orbital period of the moon", "targetFormula": "(!fb:astronomy.orbital_relationship.orbital_period fb:en.moon)"},
{"utterance": "where did barosaurus live", "targetFormula": "(fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.barosaurus)"},
{"utterance": "who wrote a series of unfortunate events", "targetFormula": "(fb:book.author.series_written_or_contributed_to fb:en.a_series_of_unfortunate_events)"},
{"utterance": "what genre is doonesbury", "targetFormula": "(!fb:comic_strips.comic_strip.genre fb:en.doonesbury)"},
{"utterance": "what religious order belongs to kirkstall abbey", "targetFormula": "(!fb:religion.monastery.religious_order fb:en.kirkstall_abbey)"},
{"utterance": "during what time period did uluru form", "targetFormula": "(!fb:geology.geological_formation.formed_during_period fb:en.uluru)"},
{"utterance": "what blogs are in german", "targetFormula": "(fb:internet.blog.language fb:en.german_language)"},
{"utterance": "what type of clothing are knickerbockers", "targetFormula": "(!fb:fashion.garment.specialization_of fb:en.knickerbockers)"},
{"utterance": "what is the address of the mitchell public library", "targetFormula": "(!fb:location.mailing_address.street_address ((lambda x (!fb:library.public_library.address (var x))) fb:m.0j9by57))"},
{"utterance": "what genres did 8 mile consist of", "targetFormula": "(!fb:film.film.genre fb:en.8_mile)"},
{"utterance": "how many types of dinosaurs were there in africa", "targetFormula": "(count (!fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.africa))"},
{"utterance": "who is the head coach of the chicago bulls", "targetFormula": "(!fb:basketball.basketball_team.head_coach fb:en.chicago_bulls)"},
{"utterance": "who did nathan fillion voice in halo: reach", "targetFormula": "(!fb:cvg.game_performance.character (and (and ((lambda x (fb:cvg.game_performance.game (var x))) fb:en.halo_reach) ((lambda x (fb:cvg.game_performance.performance_type (var x))) fb:m.02nsjvf)) ((lambda x (fb:cvg.game_performance.voice_actor (var x))) fb:en.nathan_fillion)))"},
{"utterance": "what is the motto of the order of the golden fleece", "targetFormula": "(!fb:royalty.order_of_chivalry.motto fb:en.order_of_the_golden_fleece)"},
{"utterance": "what is the retail floor space of the dubai marina mall", "targetFormula": "(!fb:business.shopping_center.retail_floor_space_m_2 fb:m.06zl3_x)"},
{"utterance": "who built the uss nautilus", "targetFormula": "(!fb:boats.ship.ship_builder fb:en.uss_nautilus)"},
{"utterance": "where did nathan smith die", "targetFormula": "(!fb:people.deceased_person.place_of_death fb:en.nathan_smith_1770)"},
{"utterance": "who owns flickr", "targetFormula": "(!fb:internet.website.owner fb:en.flickr)"},
{"utterance": "who designed the game of life", "targetFormula": "(!fb:games.game.designer fb:en.the_game_of_life)"},
{"utterance": "what medal did graeme brown receive at the 2004 summer olympics", "targetFormula": "(!fb:olympics.olympic_medal_honor.medal (and ((lambda x (fb:olympics.olympic_medal_honor.olympics (var x))) fb:en.2004_summer_olympics) ((lambda x (fb:olympics.olympic_medal_honor.medalist (var x))) fb:en.graeme_brown)))"},
{"utterance": "what is the symbol for yen", "targetFormula": "(!fb:finance.currency.prefix_symbol fb:en.japanese_yen)"},
{"utterance": "what topics are equivalent to us county", "targetFormula": "(!fb:freebase.type_profile.equivalent_topic fb:location.us_county)"},
{"utterance": "what is the origin of the mississippi river", "targetFormula": "(!fb:geography.river.origin fb:en.mississippi_river)"},
{"utterance": "how many celebrities have been charged with driving under the influence", "targetFormula": "(count (!fb:celebrities.legal_entanglement.celebrity ((lambda x (fb:celebrities.legal_entanglement.offense (var x))) fb:en.driving_under_the_influence)))"},
{"utterance": "how many expansion packs does the sims have", "targetFormula": "(count (!fb:cvg.computer_videogame.expansions fb:en.the_sims))"},
{"utterance": "what is ron glass 's religion", "targetFormula": "(!fb:people.person.religion fb:en.ron_glass)"},
{"utterance": "what is the cape may lighthouse made of", "targetFormula": "(!fb:architecture.lighthouse.construction fb:en.cape_may_lighthouse)"},
{"utterance": "what is the subject of the source report", "targetFormula": "(!fb:radio.radio_program.subjects fb:en.the_source_report)"},
{"utterance": "where was jerry seinfeld born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.jerry_seinfeld)"},
{"utterance": "what was roe v. wade about", "targetFormula": "(!fb:law.legal_case.subject fb:en.roe_v_wade)"},
{"utterance": "what position does craig adams play", "targetFormula": "(!fb:ice_hockey.hockey_player.hockey_position fb:en.craig_adams_1977)"},
{"utterance": "how many record labels was omarion under", "targetFormula": "(count (!fb:music.artist.label fb:en.omarion_grandberry))"},
{"utterance": "in 1981 what award did danny devito win", "targetFormula": "(!fb:award.award_honor.award (and ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.danny_devito) ((lambda x (fb:award.award_honor.year (var x))) (date 1981 -1 -1))))"},
{"utterance": "what sizes does pilsner urquell come in", "targetFormula": "(!fb:food.beer_containment.size ((lambda x (fb:food.beer_containment.beer (var x))) fb:en.pilsner_urquell))"},
{"utterance": "how many starring tv roles did jack albertson have", "targetFormula": "(count (!fb:tv.regular_tv_appearance.character ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.jack_albertson)))"},
{"utterance": "when was the airspeed oxford first flown", "targetFormula": "(!fb:aviation.aircraft_model.maiden_flight fb:en.airspeed_oxford)"},
{"utterance": "what is the release date for 2 fast 2 furious", "targetFormula": "(!fb:film.film.initial_release_date fb:en.2_fast_2_furious)"},
{"utterance": "who founded nivea", "targetFormula": "(!fb:organization.organization.founders fb:en.nivea)"},
{"utterance": "what are the file formats supported by the iphone", "targetFormula": "(!fb:computer.computing_platform.file_formats_supported fb:en.iphone)"},
{"utterance": "where is the tempelhof international airport located", "targetFormula": "(!fb:location.location.containedby fb:en.tempelhof_international_airport)"},
{"utterance": "what languages has microsoft designed", "targetFormula": "(!fb:computer.programming_language_designer.languages_designed fb:en.microsoft_corporation)"},
{"utterance": "what works did anthony payne finish", "targetFormula": "(!fb:media_common.completion_of_unfinished_work.unfinished_work ((lambda x (fb:media_common.completion_of_unfinished_work.finisher (var x))) fb:en.anthony_payne))"},
{"utterance": "how many awards did titanic win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.titanic_special_edition_dvd)))"},
{"utterance": "how is the plastiki propelled", "targetFormula": "(!fb:boats.ship.means_of_propulsion fb:en.plastiki)"},
{"utterance": "where was the 1st world science fiction convention held", "targetFormula": "(!fb:conferences.conference.venue fb:en.1st_world_science_fiction_convention)"},
{"utterance": "what is serge made out of", "targetFormula": "(!fb:fashion.textile.fiber fb:en.serge)"},
{"utterance": "who hosted the 2003 cricket world cup", "targetFormula": "(!fb:cricket.cricket_tournament_event.host fb:en.2003_cricket_world_cup)"},
{"utterance": "when did secretariat die", "targetFormula": "(!fb:biology.deceased_organism.date_of_death fb:en.secretariat)"},
{"utterance": "when was the galileo spacecraft launched", "targetFormula": "(!fb:spaceflight.satellite.launch_date fb:en.galileo_spacecraft)"},
{"utterance": "when was savealot founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.save-a-lot)"},
{"utterance": "how many people can play settlers of catan", "targetFormula": "(!fb:measurement_unit.integer_range.high_value (!fb:games.game.number_of_players fb:en.settlers_of_catan))"},
{"utterance": "how many exhibitions have there been about leonardo da vinci", "targetFormula": "(count (!fb:exhibitions.exhibition_subject.exhibitions_created_about_this_subject fb:en.leonardo_da_vinci))"},
{"utterance": "what buildings were destroyed in september 11th", "targetFormula": "(!fb:event.disaster.structures_destroyed fb:en.september_11_2001_attacks)"},
{"utterance": "how many dinosaur species are from africa", "targetFormula": "(count (!fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.africa))"},
{"utterance": "when was the abbey of st. gall listed as an unesco world heritage site", "targetFormula": "(!fb:protected_sites.natural_or_cultural_site_listing.date_listed (and ( (lambda x (fb:protected_sites.natural_or_cultural_site_listing.designation (var x))) fb:en.world_heritage_site) ( (lambda x (fb:protected_sites.natural_or_cultural_site_listing.listed_site (var x))) fb:en.abbey_of_st_gall)))"},
{"utterance": "what is the capacity of a rolls royce merlin", "targetFormula": "(!fb:engineering.piston_engine.capacity fb:en.rolls-royce_merlin)"},
{"utterance": "what are the side effects of morphine", "targetFormula": "(!fb:medicine.medical_treatment.side_effects fb:en.morphine)"},
{"utterance": "what is the boiling point of water", "targetFormula": "(!fb:chemistry.chemical_compound.boiling_point fb:en.cpd_c00001_h2o)"},
{"utterance": "who is the screenplay for the nutty professor by", "targetFormula": "(!fb:film.film.written_by fb:en.the_nutty_professor_1996)"},
{"utterance": "what is big daddy rated", "targetFormula": "(!fb:film.film.rating fb:en.big_daddy)"},
{"utterance": "when did michael jackson go solo", "targetFormula": "(!fb:music.artist.active_start fb:en.michael_jackson)"},
{"utterance": "when did tutankhamun die", "targetFormula": "(!fb:people.deceased_person.date_of_death fb:en.tutankhamun)"},
{"utterance": "when was the united nations founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.united_nations)"},
{"utterance": "who are the distributors for bruce almighty", "targetFormula": "(!fb:film.film_film_distributor_relationship.distributor ((lambda x (fb:film.film_film_distributor_relationship.film (var x))) fb:en.bruce_almighty))"},
{"utterance": "when was bruce almighty released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.bruce_almighty)"},
{"utterance": "what was the budget for edward scissorhands", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:film.film.estimated_budget (var x))) fb:en.edward_scissorhands))"},
{"utterance": "who owns wxpn", "targetFormula": "(!fb:broadcast.radio_station.owner fb:en.wxpn)"},
{"utterance": "what was walter gropius' architectural style", "targetFormula": "(!fb:architecture.architect.architectural_style fb:en.walter_gropius)"},
{"utterance": "how many beds does mclean hospital have", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:medicine.hospital.beds fb:en.mclean_hospital)))"},
{"utterance": "what are some gato class submarines", "targetFormula": "(!fb:boats.ship_class.ships_in_class fb:en.gato_class_submarine)"},
{"utterance": "what is the maximum altitude of stratus clouds", "targetFormula": "(!fb:meteorology.cloud.maximum_altitude_m fb:en.stratus)"},
{"utterance": "how many conferences about mathematics are there", "targetFormula": "(count (!fb:conferences.conference_subject.series_of_conferences_about_this fb:en.mathematics))"},
{"utterance": "what diet did ceratopsia have", "targetFormula": "(!fb:base.dinosaur.dinosaur.diet fb:en.ceratopsia)"},
{"utterance": "what judge presided over state v. kelly", "targetFormula": "(!fb:law.legal_case.judges fb:en.state_v_kelly)"},
{"utterance": "when was sigmod founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.sigmod)"},
{"utterance": "in what city is h&r block headquartered", "targetFormula": "(!fb:location.mailing_address.citytown ((lambda x (!fb:organization.organization.headquarters (var x))) fb:en.h_r_block))"},
{"utterance": "what is the highest elevation at the elk mountain ski area", "targetFormula": "(!fb:skiing.ski_area.top_elevation fb:en.elk_mountain_ski_area)"},
{"utterance": "what is the texture of brie cheese", "targetFormula": "(!fb:food.cheese.texture fb:m.01482f)"},
{"utterance": "how many schools have a beaver as their mascot", "targetFormula": "(count (!fb:education.school_mascot.school fb:en.beaver))"},
{"utterance": "who created sesame street", "targetFormula": "(!fb:tv.tv_program.program_creator fb:en.sesame_street)"},
{"utterance": "how many film performances does jerry seinfeld have", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.jerry_seinfeld)))"},
{"utterance": "how many spin offs does sesame street have", "targetFormula": "(count (!fb:tv.tv_program.spin_offs fb:en.sesame_street))"},
{"utterance": "what diseases cause leukemia", "targetFormula": "(!fb:medicine.disease.causes fb:en.leukemia)"},
{"utterance": "how many different products does gsusa produce", "targetFormula": "(count (!fb:business.brand.products fb:en.girl_scouts_of_the_usa))"},
{"utterance": "who directed meet the parents", "targetFormula": "(!fb:film.film.directed_by fb:en.meet_the_parents)"},
{"utterance": "when was omarion born", "targetFormula": "(!fb:people.person.date_of_birth fb:en.omarion_grandberry)"},
{"utterance": "what are some french restaurants", "targetFormula": "(!fb:dining.cuisine.restaurant fb:m.02z3r)"},
{"utterance": "who publishes the journal of abnormal psychology", "targetFormula": "(!fb:book.periodical_publisher_period.publisher ( (lambda x (fb:book.periodical_publisher_period.periodical (var x))) fb:en.journal_of_abnormal_psychology))"},
{"utterance": "what was sesame street 's original network", "targetFormula": "(!fb:tv.tv_network_duration.network ((lambda x (fb:tv.tv_network_duration.program (var x))) fb:en.sesame_street))"},
{"utterance": "what are the moons of jupiter", "targetFormula": "(!fb:astronomy.orbital_relationship.orbited_by fb:en.jupiter)"},
{"utterance": "what exhibits are there at the arignar anna zoological park", "targetFormula": "(!fb:zoos.zoo.exhibits fb:en.arignar_anna_zoological_park)"},
{"utterance": "where was neanderthal 1 found", "targetFormula": "(!fb:biology.fossil_specimen.found_at_site fb:m.026lz95)"},
{"utterance": "what radio program is a right to death an episode of", "targetFormula": "(!fb:radio.radio_program_episode.program fb:en.a_right_to_death)"},
{"utterance": "what category of martial arts is tai chi chuan", "targetFormula": "(!fb:martial_arts.martial_art.category fb:en.tai_chi_chuan)"},
{"utterance": "how many religions worship god", "targetFormula": "(count (!fb:religion.deity.deity_of fb:en.god))"},
{"utterance": "what are the subdisciplines of engineering", "targetFormula": "(!fb:education.field_of_study.subdisciplines fb:en.engineering)"},
{"utterance": "what is target 's industry", "targetFormula": "(!fb:business.business_operation.industry fb:en.target_corporation)"},
{"utterance": "how many countries use the australian dollar", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.australian_dollar))"},
{"utterance": "what software has claris developed", "targetFormula": "(!fb:computer.software_developer.software fb:en.claris)"},
{"utterance": "what bus operators run buses to paris", "targetFormula": "(!fb:travel.transportation.transport_operator ((lambda x (fb:travel.transportation.travel_destination (var x))) fb:en.paris))"},
{"utterance": "when was porgy and bess first performed", "targetFormula": "(!fb:opera.opera.date_of_first_performance fb:en.porgy_and_bess)"},
{"utterance": "how many people have been influenced by socrates", "targetFormula": "(count (!fb:influence.influence_node.influenced fb:en.socrates))"},
{"utterance": "how many runways does the san francisco international airport have", "targetFormula": "(!fb:aviation.airport.number_of_runways fb:en.san_francisco_international_airport)"},
{"utterance": "what color did adolf anderssen play in the immortal game", "targetFormula": "(!fb:chess.chess_game_participation.color (and ((lambda x (fb:chess.chess_game_participation.game (var x))) fb:en.immortal_game) ((lambda x (fb:chess.chess_game_participation.player (var x))) fb:en.adolf_anderssen)))"},
{"utterance": "when was the boston herald first published", "targetFormula": "(!fb:book.periodical_publication_date.date (!fb:book.periodical.first_issue_date fb:en.boston_herald))"},
{"utterance": "is the university of lima public or private", "targetFormula": "(!fb:education.educational_institution.school_type fb:en.university_of_lima)"},
{"utterance": "what exhibitions has the gross clinic been in", "targetFormula": "(!fb:exhibitions.exhibit.exhibitions_displayed_in fb:en.the_gross_clinic)"},
{"utterance": "what do cirrus clouds look like", "targetFormula": "(!fb:meteorology.cloud.appearance fb:m.0cs96)"},
{"utterance": "who played in the game of the century", "targetFormula": "(!fb:chess.chess_game_participation.player ((lambda x (fb:chess.chess_game_participation.game (var x))) fb:en.the_game_of_the_century))"},
{"utterance": "how many countries participated in the 2006 winter olympics", "targetFormula": "(!fb:olympics.olympic_games.number_of_countries fb:en.2006_winter_olympics)"},
{"utterance": "how many tv shows did danny devito have a leading role in", "targetFormula": "(count (!fb:tv.regular_tv_appearance.character ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.danny_devito)))"},
{"utterance": "what awards has the english patient won", "targetFormula": "(!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.the_english_patient))"},
{"utterance": "who designed the first generation ford mustang", "targetFormula": "(!fb:automotive.generation.designer fb:en.first_generation_ford_mustang)"},
{"utterance": "where is the hobbit set", "targetFormula": "(!fb:fictional_universe.work_of_fiction.setting fb:en.the_hobbit)"},
{"utterance": "who discovered neon", "targetFormula": "(!fb:chemistry.chemical_element.discoverer fb:en.neon)"},
{"utterance": "what are some british locomotive class 52 trains", "targetFormula": "(!fb:rail.locomotive_class.locomotives_of_this_class fb:en.british_rail_class_52)"},
{"utterance": "what number was david akers on the philadelphia eagles", "targetFormula": "(!fb:american_football.football_historical_roster_position.number (and ( (lambda x (fb:american_football.football_historical_roster_position.team (var x))) fb:en.philadelphia_eagles) ( (lambda x (fb:american_football.football_historical_roster_position.player (var x))) fb:en.david_akers)))"},
{"utterance": "how many shows was hayden panettiere on", "targetFormula": "(count (!fb:tv.regular_tv_appearance.character ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.hayden_panettiere)))"},
{"utterance": "how many free exhibits have there been at the science museum", "targetFormula": "(count (!fb:exhibitions.exhibition_run.exhibition ((lambda x (fb:exhibitions.exhibition_run.venue (var x))) fb:en.science_museum_great_britain)))"},
{"utterance": "how did samuel beckett die", "targetFormula": "(!fb:people.deceased_person.cause_of_death fb:en.samuel_beckett)"},
{"utterance": "what type of infection is syphilis", "targetFormula": "(!fb:medicine.infectious_disease.infectious_agent_type fb:en.syphillis)"},
{"utterance": "what is africa 's area", "targetFormula": "(!fb:location.location.area fb:en.africa)"},
{"utterance": "how long are adventure time episodes", "targetFormula": "(!fb:tv.tv_program.episode_running_time fb:m.0280hsm)"},
{"utterance": "who designed the hawker sea hawk", "targetFormula": "(!fb:aviation.aircraft_model.designed_by fb:en.hawker_sea_hawk)"},
{"utterance": "when was the high court of australia founded", "targetFormula": "(!fb:law.court.founded fb:en.high_court_of_australia)"},
{"utterance": "who brews pauwel kwak", "targetFormula": "(!fb:food.beer.brewery_brand fb:en.pauwel_kwak)"},
{"utterance": "when did everything is illuminated win the guardian first book award", "targetFormula": "(!fb:award.award_honor.year (and ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.everything_is_illuminated) ((lambda x (fb:award.award_honor.award (var x))) fb:en.guardian_first_book_award)))"},
{"utterance": "what airlines is the london heathrow airport a hub for", "targetFormula": "(!fb:aviation.airport.hub_for fb:en.london_heathrow_airport)"},
{"utterance": "who are some notable agnostic figures", "targetFormula": "(!fb:religion.religion.notable_figures fb:en.agnosticism)"},
{"utterance": "what lines does the london overground operate", "targetFormula": "(!fb:rail.railway_operator_relationship.railway ((lambda x (fb:rail.railway_operator_relationship.operator (var x))) fb:en.london_overground))"},
{"utterance": "how many judges has the high court of justice had", "targetFormula": "(count (!fb:law.judicial_tenure.judge ((lambda x (fb:law.judicial_tenure.court (var x))) fb:en.high_court_of_justice)))"},
{"utterance": "what other names is the united nations known by", "targetFormula": "(!fb:common.topic.alias fb:en.united_nations)"},
{"utterance": "where did taekwondo originate", "targetFormula": "(!fb:martial_arts.martial_art.origin fb:en.taekwondo)"},
{"utterance": "what is the area of north america", "targetFormula": "(!fb:location.location.area fb:en.north_america)"},
{"utterance": "since when has the arabic alphabet been used", "targetFormula": "(!fb:language.language_writing_system.used_from fb:en.arabic_alphabet)"},
{"utterance": "how long is the mexican federal highway 1", "targetFormula": "(!fb:transportation.road.length fb:en.mexican_federal_highway_1)"},
{"utterance": "how many schools have a chapter of sigma phi epsilon", "targetFormula": "(count (!fb:education.fraternity_sorority.colleges_and_universities fb:en.sigma_phi_epsilon))"},
{"utterance": "who were the astronauts in the apollo 12 mission", "targetFormula": "(!fb:spaceflight.space_mission.astronauts fb:en.apollo_12)"},
{"utterance": "how many people influenced karl marx", "targetFormula": "(count (!fb:influence.influence_node.influenced_by fb:en.karl_marx))"},
{"utterance": "how many canon digital cameras are there", "targetFormula": "(count (!fb:digicams.digital_camera_manufacturer.cameras fb:en.canon))"},
{"utterance": "what is the original language of the wind-up bird chronicle", "targetFormula": "(!fb:book.written_work.original_language fb:en.the_wind-up_bird_chronicle)"},
{"utterance": "what is the usa money currency code", "targetFormula": "(!fb:finance.currency.currency_code fb:en.us)"},
{"utterance": "how many military units originated in delaware", "targetFormula": "(count (!fb:military.military_unit_place_of_origin.military_units fb:en.delaware))"},
{"utterance": "how many people have held the title of grand master of the order of the most holy annunciation", "targetFormula": "(count (!fb:royalty.chivalric_order_position_tenure.officer (and ( (lambda x (fb:royalty.chivalric_order_position_tenure.chivalric_office (var x))) fb:en.grand_master) ( (lambda x (fb:royalty.chivalric_order_position_tenure.order (var x))) fb:en.order_of_the_most_holy_annunciation))))"},
{"utterance": "how many malls does oxford properties own", "targetFormula": "(count (!fb:business.shopping_center_owner.shopping_centers_owned fb:en.oxford_properties))"},
{"utterance": "what qualification does woody strode have in seishindo kenpo", "targetFormula": "(!fb:martial_arts.martial_arts_certification.qualification (and ((lambda x (fb:martial_arts.martial_arts_certification.art (var x))) fb:en.seishindo_kenpo) ((lambda x (fb:martial_arts.martial_arts_certification.person (var x))) fb:en.woody_strode)))"},
{"utterance": "what is the single letter abreviation for tryptophan", "targetFormula": "(!fb:biology.amino_acid.single_letter_abbreviation fb:en.tryptophan)"},
{"utterance": "where are dravidian languages spoken", "targetFormula": "(!fb:language.language_family.geographic_distribution fb:en.dravidian_languages)"},
{"utterance": "how many books does the detroit public library have", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:library.public_library_system.collection_size fb:en.detroit_public_library)))"},
{"utterance": "how many kinds of footwear are there", "targetFormula": "(count (!fb:fashion.garment.more_specialized_forms fb:en.footwear))"},
{"utterance": "in what park is splash mountain", "targetFormula": "(fb:amusement_parks.park.rides fb:en.splash_mountain)"},
{"utterance": "what is danny devito 's religion", "targetFormula": "(!fb:people.person.religion fb:en.danny_devito)"},
{"utterance": "what powers does sonic the hedgehog have", "targetFormula": "(!fb:fictional_universe.fictional_character.powers_or_abilities fb:en.sonic_the_hedgehog)"},
{"utterance": "how many wins has larry bird had in his career", "targetFormula": "(!fb:basketball.basketball_coach.season_wins fb:en.larry_bird)"},
{"utterance": "what are the symptoms of motor neuron disease", "targetFormula": "(!fb:medicine.disease.symptoms fb:en.motor_neuron_disease)"},
{"utterance": "what was the final air date of lux radio theater", "targetFormula": "(!fb:radio.radio_program.final_broadcast fb:en.lux_radio_theater)"},
{"utterance": "how many grammy awards did beyonce receive in her singing career", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.beyonce)))"},
{"utterance": "what language is lohengrin in", "targetFormula": "(!fb:opera.opera.language fb:m.09hvx)"},
{"utterance": "which issues of x-men have iceman on the cover", "targetFormula": "(!fb:comic_books.comic_book_character.cover_appearances fb:en.iceman)"},
{"utterance": "what number is kevin youkilis on the boston red sox", "targetFormula": "(!fb:sports.sports_team_roster.number (and ((lambda x (fb:sports.sports_team_roster.player (var x))) fb:en.kevin_youkilis) ((lambda x (fb:sports.sports_team_roster.team (var x))) fb:en.boston_red_sox)))"},
{"utterance": "what are the symptoms of prostate cancer", "targetFormula": "(!fb:medicine.disease.symptoms fb:en.prostate_cancer)"},
{"utterance": "who created charlie brown", "targetFormula": "(!fb:fictional_universe.fictional_character.character_created_by fb:en.charlie_brown)"},
{"utterance": "what class of train was heavy harry", "targetFormula": "(!fb:rail.locomotive.locomotive_class fb:m.0glzm0h)"},
{"utterance": "what type of bicycle is a bianchi pista", "targetFormula": "(!fb:bicycles.bicycle_model.bicycle_type fb:en.bianchi_pista)"},
{"utterance": "how many broadcasts does blogtalkradio distribute", "targetFormula": "(count (!fb:broadcast.distributor.distributes fb:en.blogtalkradio))"},
{"utterance": "what label does julien fournie design for", "targetFormula": "(!fb:fashion.designer_label_association.label ((lambda x (fb:fashion.designer_label_association.designer (var x))) fb:en.julien_fournie))"},
{"utterance": "how high is niagara falls", "targetFormula": "(!fb:geography.waterfall.height fb:en.niagara_falls)"},
{"utterance": "who currently plays for the san jose sharks", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.player ((lambda x (fb:ice_hockey.hockey_roster_position.team (var x))) fb:en.san_jose_sharks))"},
{"utterance": "what are the survival rates for prostate cancer", "targetFormula": "(!fb:medicine.survival_rate.rate ((lambda x (!fb:medicine.disease.survival_rates (var x))) fb:en.prostate_cancer))"},
{"utterance": "how many award nominations did danny devito have", "targetFormula": "(count (!fb:award.award_nomination.award ((lambda x (fb:award.award_nomination.award_nominee (var x))) fb:en.danny_devito)))"},
{"utterance": "when did charlie sheen enter the thousand oaks rehab facility", "targetFormula": "(!fb:celebrities.rehab.entered (and ((lambda x (fb:celebrities.rehab.rehab_facility (var x))) fb:m.04fz39z) ((lambda x (fb:celebrities.rehab.celebrity (var x))) fb:en.charlie_sheen)))"},
{"utterance": "what bowling technique does matthew hayden use", "targetFormula": "(!fb:cricket.cricket_bowler.technique fb:en.matthew_hayden)"},
{"utterance": "what's the focus of the last minute blog", "targetFormula": "(!fb:internet.blog.focus fb:en.the_last_minute_blog)"},
{"utterance": "how many nicknames does shaquille o'neal have", "targetFormula": "(count (!fb:common.topic.alias fb:en.shaquille_oneal))"},
{"utterance": "in what division do the st. louis blues play", "targetFormula": "(!fb:ice_hockey.hockey_team.division fb:m.06x6s)"},
{"utterance": "who managed liverpool f.c. from 2004 to june 2010", "targetFormula": "(!fb:soccer.football_team_management_tenure.manager (and ((lambda x (fb:soccer.football_team_management_tenure.to (var x))) (date 2004 -1 -1)) ((lambda x (fb:soccer.football_team_management_tenure.team (var x))) fb:en.liverpool_fc)))"},
{"utterance": "what is the height restriction on the roller soaker", "targetFormula": "(!fb:amusement_parks.ride.height_restriction fb:en.roller_soaker)"},
{"utterance": "what is currency code for the canadian dollar", "targetFormula": "(!fb:finance.currency.currency_code fb:en.canadian_dollar)"},
{"utterance": "what is the area of antarctica", "targetFormula": "(!fb:location.location.area fb:en.antarctica)"},
{"utterance": "how many games did donovan mcnabb play in the 2008 season", "targetFormula": "(!fb:american_football.player_game_statistics.games (and ((lambda x (fb:american_football.player_game_statistics.season (var x))) fb:en.2008_nfl_season) ((lambda x (fb:american_football.player_game_statistics.player (var x))) fb:en.donovan_mcnabb)))"},
{"utterance": "how many issues are there in the x-men", "targetFormula": "(!fb:comic_books.comic_book_series.number_of_issues fb:m.02hqp38)"},
{"utterance": "in how many countries is the euro used", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.euro))"},
{"utterance": "what was target 's rank in the fortune 500 in 2010", "targetFormula": "(!fb:award.ranking.rank (and (and ((lambda x (fb:award.ranking.list (var x))) fb:en.fortune_500) ((lambda x (fb:award.ranking.year (var x))) (date 2010 -1 -1))) ((lambda x (fb:award.ranking.item (var x))) fb:en.target_corporation)))"},
{"utterance": "how many people were at the 2006 fifa world cup final", "targetFormula": "(count (!fb:soccer.football_match.attendance fb:en.2006_fifa_world_cup_final))"},
{"utterance": "how many awards did danny devito win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.danny_devito)))"},
{"utterance": "how many islands are in the antilles", "targetFormula": "(count (!fb:geography.island_group.islands_in_group fb:en.antilles))"},
{"utterance": "what is the closet city to snowshoe mountain", "targetFormula": "(!fb:skiing.ski_area.closest_city fb:en.snowshoe_mountain)"},
{"utterance": "how tall is jerry seinfeld", "targetFormula": "(!fb:people.person.height_meters fb:en.jerry_seinfeld)"},
{"utterance": "how many cards do you need to play canasta", "targetFormula": "(!fb:games.playing_card_game.number_of_cards fb:en.canasta)"},
{"utterance": "how many games are in the final fantasy series", "targetFormula": "(count (!fb:cvg.game_series.games_in_series fb:en.final_fantasy))"},
{"utterance": "what are nicknames for new york", "targetFormula": "(!fb:common.topic.alias fb:en.new_york_state)"},
{"utterance": "what is the highway fuel economy of a 2007 honda civic hybrid", "targetFormula": "(!fb:automotive.us_fuel_economy.highway_mpg ((lambda x (fb:automotive.us_fuel_economy.trim_level (var x))) fb:m.04nbbrp))"},
{"utterance": "who is the developer of java language", "targetFormula": "(!fb:computer.programming_language.developers fb:m.07sbkfb)"},
{"utterance": "how many speeds does a panasonic dx3000 have", "targetFormula": "(!fb:bicycles.bicycle_model.speeds fb:en.panasonic_dx3000)"},
{"utterance": "when was joni mitchell inducted into the rock and roll hall of fame", "targetFormula": "(!fb:award.hall_of_fame_induction.date (and ((lambda x (fb:award.hall_of_fame_induction.inductee (var x))) fb:en.joni_mitchell) ((lambda x (fb:award.hall_of_fame_induction.hall_of_fame (var x))) fb:en.rock_and_roll_hall_of_fame)))"},
{"utterance": "where was william shakespeare born", "targetFormula": "(!fb:people.person.place_of_birth fb:en.william_shakespeare)"},
{"utterance": "what are the dimensions of the matrix reloaded image", "targetFormula": "(!fb:common.image.size fb:m.0kjl0q)"},
{"utterance": "how many test conferences has rudi koertzen refereed", "targetFormula": "(!fb:cricket.cricket_umpire.test_matches_refereed fb:en.rudi_koertzen)"},
{"utterance": "what religions branched from roman catholicism", "targetFormula": "(!fb:religion.religion.branched_into fb:en.roman_catholicism)"},
{"utterance": "what area does kifm serve", "targetFormula": "(!fb:broadcast.radio_station.serves_area fb:en.kifm)"},
{"utterance": "who developed the polio vaccine", "targetFormula": "(!fb:medicine.vaccine.developed_by fb:en.polio_vaccine)"},
{"utterance": "who painted the mona lisa", "targetFormula": "(!fb:visual_art.artwork.artist fb:en.mona_lisa)"},
{"utterance": "how many collections did barry halper have", "targetFormula": "(count (!fb:interests.collection.category ((lambda x (fb:interests.collection.collector (var x))) fb:en.barry_halper)))"},
{"utterance": "who writes the dabble blog", "targetFormula": "(!fb:internet.blog.blogger fb:en.the_dabble_blog)"},
{"utterance": "what conventions has hillary clinton spoken at", "targetFormula": "(!fb:base.politicalconventions.convention_speech.venue ( (lambda x (fb:base.politicalconventions.convention_speech.speaker (var x))) fb:en.hillary_rodham_clinton))"},
{"utterance": "who are some bauhaus architects", "targetFormula": "(!fb:architecture.architectural_style.architects fb:en.international_style)"},
{"utterance": "how many teams has curly lambeau coached", "targetFormula": "(count (!fb:american_football.football_historical_coach_position.team ( (lambda x (fb:american_football.football_historical_coach_position.coach (var x))) fb:en.curly_lambeau)))"},
{"utterance": "how many branches does the vancouver public library have", "targetFormula": "(count (!fb:library.public_library_system.branches fb:en.vancouver_public_library))"},
{"utterance": "what city is the ben may main library in", "targetFormula": "(!fb:location.mailing_address.citytown ((lambda x (!fb:library.public_library.address (var x))) fb:m.02ncllz))"},
{"utterance": "what guidebooks are there for san francisco", "targetFormula": "(!fb:travel.travel_destination.guidebooks fb:en.san_francisco)"},
{"utterance": "what are some psychology journals", "targetFormula": "(!fb:education.field_of_study.journals_in_this_discipline fb:en.psychology)"},
{"utterance": "where was the kepler spacecraft launched", "targetFormula": "(!fb:spaceflight.satellite.launch_site fb:en.kepler_mission)"},
{"utterance": "how many film actors are there on freebase", "targetFormula": "(!fb:freebase.type_profile.instance_count fb:film.actor)"},
{"utterance": "when did a prairie home companion first air", "targetFormula": "(!fb:radio.radio_program.first_broadcast fb:en.a_prairie_home_companion)"},
{"utterance": "what genres does meet the parents consist of", "targetFormula": "(!fb:film.film.genre fb:en.meet_the_parents)"},
{"utterance": "what is the max speed of a gloster meteor", "targetFormula": "(!fb:aviation.aircraft_model.maximum_speed_km_h fb:en.gloster_meteor)"},
{"utterance": "what position does colby armstrong play on the toronto maple leafs", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.position (and ((lambda x (fb:ice_hockey.hockey_roster_position.player (var x))) fb:en.colby_armstrong) ((lambda x (fb:ice_hockey.hockey_roster_position.team (var x))) fb:en.toronto_maple_leafs)))"},
{"utterance": "how many countries are in the european union", "targetFormula": "(count (!fb:organization.organization_membership.member ((lambda x (fb:organization.organization_membership.organization (var x))) fb:en.european_union)))"},
{"utterance": "when was civilization first released", "targetFormula": "(!fb:cvg.computer_videogame.release_date fb:m.01vvf)"},
{"utterance": "on what network was gilligan_s island first aired", "targetFormula": "(!fb:tv.tv_network_duration.network ((lambda x (fb:tv.tv_network_duration.program (var x))) fb:en.gilligans_island))"},
{"utterance": "what series did tamora pierce write", "targetFormula": "(!fb:book.author.series_written_or_contributed_to fb:en.tamora_pierce)"},
{"utterance": "who presented the gettysburg address", "targetFormula": "(!fb:event.speech_or_presentation.speaker_s ((lambda x (fb:event.speech_or_presentation.presented_work (var x))) fb:en.gettysburg_address))"},
{"utterance": "what is the mascot of the boston red sox", "targetFormula": "(!fb:sports.sports_team.team_mascot fb:en.boston_red_sox)"},
{"utterance": "when was cathy landers certified as a fifth degree black belt in seishindo kenpo", "targetFormula": "(!fb:martial_arts.martial_arts_certification.date (and (and ((lambda x (fb:martial_arts.martial_arts_certification.qualification (var x))) fb:en.fifth_degree) ((lambda x (fb:martial_arts.martial_arts_certification.art (var x))) fb:en.seishindo_kenpo)) ((lambda x (fb:martial_arts.martial_arts_certification.person (var x))) fb:en.cathy_landers)))"},
{"utterance": "how many locations are there on freebase", "targetFormula": "(!fb:freebase.type_profile.instance_count fb:location.location)"},
{"utterance": "who are gatorade 's major sponsors", "targetFormula": "(!fb:business.sponsorship.sponsored_recipient ((lambda x (fb:business.sponsorship.sponsored_by (var x))) fb:en.gatorade))"},
{"utterance": "what is the fuel economy of a 2008 hyundai accent gls sedan in the city", "targetFormula": "(!fb:automotive.us_fuel_economy.highway_mpg ((lambda x (fb:automotive.us_fuel_economy.trim_level (var x))) fb:m.04nbpg5))"},
{"utterance": "where is between the lines produced", "targetFormula": "(!fb:broadcast.content.location fb:m.03d5yt5)"},
{"utterance": "when was the oreo cookie introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.oreo))"},
{"utterance": "what positon does ray lewis currently play for the baltimore ravens", "targetFormula": "(!fb:american_football.football_historical_roster_position.position_s (and ( (lambda x (fb:american_football.football_historical_roster_position.player (var x))) fb:en.ray_lewis) ( (lambda x (fb:american_football.football_historical_roster_position.team (var x))) fb:en.baltimore_ravens)))"},
{"utterance": "to whom is invisible man dedicated", "targetFormula": "(!fb:media_common.dedication.dedicated_to ((lambda x (fb:media_common.dedication.work_dedicated (var x))) fb:en.invisible_man))"},
{"utterance": "how is syphilis transmitted", "targetFormula": "(!fb:medicine.infectious_disease.transmission fb:en.syphillis)"},
{"utterance": "what characters sing soprano in tristan and isolde", "targetFormula": "(!fb:opera.opera_character_voice.character (and ((lambda x (fb:opera.opera_character_voice.opera (var x))) fb:en.tristan_und_isolde) ((lambda x (fb:opera.opera_character_voice.voice (var x))) fb:en.crystal_clear_soprano)))"},
{"utterance": "what team did rick carlisle coach in 2002", "targetFormula": "(!fb:basketball.basketball_historical_coach_position.team (and ((lambda x (fb:basketball.basketball_historical_coach_position.from (var x))) (date 2002 -1 -1)) ((lambda x (fb:basketball.basketball_historical_coach_position.coach (var x))) fb:en.rick_carlisle)))"},
{"utterance": "how many dialects of arabic are there", "targetFormula": "(count (!fb:language.human_language.dialects fb:en.arabic_language))"},
{"utterance": "when did invertigo open", "targetFormula": "(!fb:amusement_parks.ride.opened fb:m.0flmt0)"},
{"utterance": "what is the melting point of gold", "targetFormula": "(!fb:chemistry.chemical_element.melting_point fb:m.025rs2z)"},
{"utterance": "what is the thinker made of", "targetFormula": "(!fb:visual_art.artwork.media fb:en.the_thinker)"},
{"utterance": "who did jun matsumoto play in hana yori dango", "targetFormula": "(!fb:tv.regular_tv_appearance.character (and ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.jun_matsumoto) ((lambda x (fb:tv.regular_tv_appearance.series (var x))) fb:m.05f52tw)))"},
{"utterance": "how many cars are in jay leno's car collection", "targetFormula": "(count (!fb:interests.collection.items (and ((lambda x (fb:interests.collection.collector (var x))) fb:en.jay_leno) ((lambda x (fb:interests.collection.category (var x))) fb:en.automobile))))"},
{"utterance": "when did coronet peak open", "targetFormula": "(!fb:skiing.ski_area.opening_date fb:en.coronet_peak)"},
{"utterance": "what series was game of thrones adapted from", "targetFormula": "(!fb:media_common.adaptation.adapted_from fb:en.game_of_thrones)"},
{"utterance": "what was the science of survival exhibit about", "targetFormula": "(!fb:exhibitions.exhibition.subjects fb:m.046chvq)"},
{"utterance": "how many legal offences has lindsey lohan committed", "targetFormula": "(count (!fb:celebrities.legal_entanglement.location ((lambda x (fb:celebrities.legal_entanglement.celebrity (var x))) fb:en.lindsay_lohan)))"},
{"utterance": "how many people visit the new york public library annually", "targetFormula": "(count (!fb:measurement_unit.dated_integer.number (!fb:library.public_library_system.annual_visits fb:en.new_york_public_library)))"},
{"utterance": "how many comic stories has adrienne roy colored", "targetFormula": "(count (!fb:comic_books.comic_book_colorist.comic_stories_colored fb:en.adrienne_roy))"},
{"utterance": "what games has garry kasparov won", "targetFormula": "(!fb:chess.chess_game_participation.game ((lambda x (fb:chess.chess_game_participation.outcome (var x))) fb:m.0454vkd))"},
{"utterance": "how tall do you have to be to ride the american eagle", "targetFormula": "(!fb:amusement_parks.ride.height_restriction fb:m.0cdlng)"},
{"utterance": "\ufeffwho is the head coach of the pittsburgh steelers", "targetFormula": "(!fb:american_football.football_team.current_head_coach fb:en.pittsburgh_steelers)"},
{"utterance": "what conditions does aspirin treat", "targetFormula": "(!fb:medicine.medical_treatment.used_to_treat fb:en.aspirin)"},
{"utterance": "for what team does fedor tyutin play", "targetFormula": "(!fb:ice_hockey.hockey_roster_position.team ((lambda x (fb:ice_hockey.hockey_roster_position.player (var x))) fb:en.fedor_tyutin))"},
{"utterance": "how many tv shows was ron glass in", "targetFormula": "(count (!fb:tv.regular_tv_appearance.series ((lambda x (fb:tv.regular_tv_appearance.actor (var x))) fb:en.ron_glass)))"},
{"utterance": "how many ships were built by bath iron works", "targetFormula": "(count (!fb:boats.ship_builder.ships_built fb:en.bath_iron_works))"},
{"utterance": "who was nominated for the academy award for best director in 2011", "targetFormula": "(!fb:award.award_nomination.award_nominee (and ((lambda x (fb:award.award_nomination.award (var x))) fb:en.academy_award_for_best_director) ((lambda x (fb:award.award_nomination.year (var x))) (date 2011 -1 -1))))"},
{"utterance": "what character says \"i'll be back\"", "targetFormula": "(!fb:media_common.quotation.spoken_by_character fb:m.06zh9s)"},
{"utterance": "what is britney spears' sexual orientation", "targetFormula": "(!fb:celebrities.sexual_orientation_phase.sexual_orientation ((lambda x (fb:celebrities.sexual_orientation_phase.celebrity (var x))) fb:en.britney_spears))"},
{"utterance": "how many engines have a straight-4 piston configuration", "targetFormula": "(count (!fb:engineering.piston_configuration.engines fb:en.straight-4))"},
{"utterance": "what players scored in the 2006 fifa world cup final", "targetFormula": "(!fb:soccer.football_goal.scorer ((lambda x (fb:soccer.football_goal.match (var x))) fb:en.2006_fifa_world_cup_final))"},
{"utterance": "how many athletes have gotten an olympic gold medal", "targetFormula": "(count (!fb:olympics.olympic_medal_honor.medalist ((lambda x (fb:olympics.olympic_medal_honor.medal (var x))) fb:en.gold_medal)))"},
{"utterance": "what movies were nominated for favorite comedy movie", "targetFormula": "(!fb:award.award_nomination.nominated_for ( (lambda x (fb:award.award_nomination.award (var x))) fb:en.peoples_choice_award_for_favorite_comedy_movie))"},
{"utterance": "who lived in fallingwater", "targetFormula": "(!fb:architecture.occupancy.occupant ((lambda x (fb:architecture.occupancy.building (var x))) fb:en.fallingwater))"},
{"utterance": "who developed capri sun", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.capri_sun))"},
{"utterance": "what lodges are in the alta ski area", "targetFormula": "(!fb:skiing.ski_area.lodges fb:en.alta_ski_area)"},
{"utterance": "does the canon powershot a75 have an orientation sensor", "targetFormula": "(!fb:digicams.digital_camera.orientation_sensor fb:en.canon_powershot_a75)"},
{"utterance": "how long is titanic 's runtime", "targetFormula": "(!fb:film.film_cut.runtime ((lambda x (fb:film.film_cut.film (var x))) fb:en.titanic_special_edition_dvd))"},
{"utterance": "what team does mike brown coach", "targetFormula": "(!fb:basketball.basketball_coach.team fb:en.mike_brown_1970)"},
{"utterance": "what team does joe girardi currently manage", "targetFormula": "(!fb:baseball.baseball_manager.current_team_managed fb:en.joe_girardi)"},
{"utterance": "when was country time produced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.country_time))"},
{"utterance": "who is the ceo of apple", "targetFormula": "(!fb:organization.leadership.person (and ((lambda x (fb:organization.leadership.role (var x))) fb:en.chief_executive_officer) ((lambda x (fb:organization.leadership.organization (var x))) fb:en.apple_inc)))"},
{"utterance": "how many commanders were there in the civil war", "targetFormula": "(count (!fb:military.military_command.military_commander ((lambda x (fb:military.military_command.military_conflict (var x))) fb:en.american_civil_war)))"},
{"utterance": "when did francesco sabatini start working on the puerta de san vicente", "targetFormula": "(!fb:projects.project_participation.from_date (and ((lambda x (fb:projects.project_participation.project (var x))) fb:m.0gk9x46) ((lambda x (fb:projects.project_participation.participant (var x))) fb:en.francesco_sabatini)))"},
{"utterance": "when did martin margiela start designing for herm", "targetFormula": "(!fb:fashion.designer_label_association.from_date (and ((lambda x (fb:fashion.designer_label_association.designer (var x))) fb:en.martin_margiela) ((lambda x (fb:fashion.designer_label_association.label (var x))) fb:m.06dr8q)))"},
{"utterance": "who directed 2 fast 2 furious", "targetFormula": "(!fb:film.film.directed_by fb:en.2_fast_2_furious)"},
{"utterance": "how many languages has jrr tolkein created", "targetFormula": "(count (!fb:language.language_creator.languages_created fb:en.j_r_r_tolkien))"},
{"utterance": "what telescopes does the lowell observatory have", "targetFormula": "(!fb:astronomy.astronomical_observatory.telescope_s fb:en.lowell_observatory)"},
{"utterance": "how long is the tokyu tamagawa line", "targetFormula": "(!fb:rail.railway.length fb:en.tokyu_tamagawa_line)"},
{"utterance": "who was the newscaster in 1948 on cbs evening news", "targetFormula": "(!fb:tv.tv_regular_personal_appearance.person (and (and ((lambda x (fb:tv.tv_regular_personal_appearance.appearance_type (var x))) fb:en.newscaster) ((lambda x (fb:tv.tv_regular_personal_appearance.program (var x))) fb:en.cbs_evening_news)) ((lambda x (fb:tv.tv_regular_personal_appearance.from (var x))) (date 1948 -1 -1))))"},
{"utterance": "who is the music by in bruce almighty", "targetFormula": "(!fb:film.film.music fb:en.bruce_almighty)"},
{"utterance": "how many episodes does sesame street have", "targetFormula": "(count (!fb:tv.tv_program.episodes fb:en.sesame_street))"},
{"utterance": "what is the area of australia", "targetFormula": "(!fb:location.location.area fb:en.australia)"},
{"utterance": "what is the chemical formula for butane", "targetFormula": "(!fb:chemistry.chemical_compound.formula fb:en.butane)"},
{"utterance": "how many seasons of sesame street are there", "targetFormula": "(count (!fb:tv.tv_program.seasons fb:en.sesame_street))"},
{"utterance": "who created superman", "targetFormula": "(!fb:comic_books.comic_book_character.created_by fb:en.superman)"},
{"utterance": "what is the runtime for meet the parents", "targetFormula": "(!fb:film.film_cut.runtime ((lambda x (fb:film.film_cut.film (var x))) fb:en.meet_the_parents))"},
{"utterance": "who was the casting director for meet the parents", "targetFormula": "(!fb:film.film.film_casting_director fb:en.meet_the_parents)"},
{"utterance": "how many generations of the ipod are there", "targetFormula": "(count (!fb:business.consumer_product.product_line fb:en.ipod))"},
{"utterance": "how many awards did jack albertson win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.jack_albertson)))"},
{"utterance": "since when has the california gull been the state bird of utah", "targetFormula": "(!fb:location.location_symbol_relationship.date_adopted (and (and ((lambda x (fb:location.location_symbol_relationship.administrative_division (var x))) fb:en.utah) ((lambda x (fb:location.location_symbol_relationship.Kind_of_symbol (var x))) fb:en.state_bird)) ((lambda x (fb:location.location_symbol_relationship.symbol (var x))) fb:en.california_gull)))"},
{"utterance": "what are the new york giants also known as", "targetFormula": "(!fb:common.topic.alias fb:en.new_york_giants)"},
{"utterance": "when was universal studios japan opened", "targetFormula": "(!fb:amusement_parks.park.opened fb:en.universal_studios_japan)"},
{"utterance": "how many people worked on the design and construction of the golden gate bridge", "targetFormula": "(count (!fb:projects.project_participation.participant ( (lambda x (fb:projects.project_participation.project (var x))) fb:en.design_and_construction_of_the_golden_gate_bridge)))"},
{"utterance": "does doom have a multiplayer mode", "targetFormula": "(and fb:en.doom (fb:cvg.computer_videogame.gameplay_modes fb:en.multiplayer_game))"},
{"utterance": "when was starbucks founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.starbucks)"},
{"utterance": "how much money in damages did the 1904 toronto fire cost", "targetFormula": "(!fb:measurement_unit.money_value.amount (!fb:event.disaster.damage fb:en.1904_toronto_fire))"},
{"utterance": "what newspapers does carnegie mellon publish", "targetFormula": "(!fb:education.educational_institution.newspaper fb:en.carnegie_mellon_university)"},
{"utterance": "how many schools are in the school district of philadelphia", "targetFormula": "(count (!fb:education.school_district.schools fb:en.school_district_of_philadelphia))"},
{"utterance": "what rehab facilities has jerry garcia gone to", "targetFormula": "(!fb:celebrities.rehab.rehab_facility ((lambda x (fb:celebrities.rehab.celebrity (var x))) fb:en.jerry_garcia))"},
{"utterance": "how many peabody award winners are there", "targetFormula": "(count (!fb:award.award_honor.award_winner ((lambda x (fb:award.award_honor.award (var x))) fb:en.peabody_award)))"},
{"utterance": "how many games are there for nintendo ds", "targetFormula": "(count (!fb:cvg.cvg_platform.games_on_this_platform fb:en.nintendo_ds))"},
{"utterance": "who invented scrabble", "targetFormula": "(!fb:law.invention.inventor fb:en.scrabble)"},
{"utterance": "where was the armory show exhibited", "targetFormula": "(!fb:exhibitions.exhibition_run.venue ((lambda x (fb:exhibitions.exhibition_run.exhibition (var x))) fb:en.armory_show))"},
{"utterance": "what stadium do the phillies play in", "targetFormula": "(!fb:sports.sports_team.arena_stadium fb:en.philadelphia_phillies)"},
{"utterance": "who was the director of the nutty professor", "targetFormula": "(!fb:film.film.directed_by fb:en.the_nutty_professor_1996)"},
{"utterance": "what is the spin of a positron", "targetFormula": "(!fb:physics.particle.spin fb:en.positron)"},
{"utterance": "how many brands does sara lee own", "targetFormula": "(count (!fb:business.company_brand_relationship.brand ((lambda x (fb:business.company_brand_relationship.company (var x))) fb:en.sara_lee)))"},
{"utterance": "who was the 22nd president", "targetFormula": "(fb:government.us_president.presidency_number (number 22.0 fb:en.unitless))"},
{"utterance": "what genre is the hound of the baskervilles", "targetFormula": "(fb:media_common.literary_genre.books_in_this_genre fb:en.the_hound_of_the_baskervilles)"},
{"utterance": "what is the dry mass of the hubble space telescope", "targetFormula": "(!fb:spaceflight.satellite.dry_mass_kg fb:en.hubble_space_telescope)"},
{"utterance": "what are some examples of a road bike", "targetFormula": "(!fb:bicycles.bicycle_type.bicycle_models_of_this_type fb:en.road_bicycle)"},
{"utterance": "what particles are in the lepton family", "targetFormula": "(!fb:physics.particle_family.particles fb:en.lepton)"},
{"utterance": "what category does the film domain belong to", "targetFormula": "(!fb:freebase.domain_profile.category fb:m.010s)"},
{"utterance": "what rock is gruta das torres made of", "targetFormula": "(!fb:geology.geological_formation.type_of_rock fb:en.gruta_das_torres)"},
{"utterance": "what processor did the apple ii use", "targetFormula": "(!fb:computer.computer.processor fb:en.apple_ii)"},
{"utterance": "when is jerry seinfeld 's birthday", "targetFormula": "(!fb:people.person.date_of_birth fb:en.jerry_seinfeld)"},
{"utterance": "what language is tagliablog in", "targetFormula": "(!fb:internet.blog.language fb:en.tagliablog)"},
{"utterance": "who produced the three doctors", "targetFormula": "(!fb:tv.tv_producer_episode_credit.producer (!fb:tv.tv_series_episode.producers fb:en.the_three_doctors))"},
{"utterance": "who manufactured the space shuttle discovery", "targetFormula": "(!fb:spaceflight.spacecraft.manufacturer fb:en.space_shuttle_discovery)"},
{"utterance": "what is the population of asia", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.asia))"},
{"utterance": "how many areas were affected by hurricane ivan", "targetFormula": "(count (!fb:meteorology.tropical_cyclone.affected_areas fb:en.hurricane_ivan))"},
{"utterance": "when was kid_s corner first broadcast", "targetFormula": "(!fb:radio.radio_program.first_broadcast fb:m.05v47f_)"},
{"utterance": "what is currency code for the mexican peso", "targetFormula": "(!fb:finance.currency.currency_code fb:en.mexican_peso)"},
{"utterance": "what languages use bengali script", "targetFormula": "(!fb:language.language_writing_system.languages fb:en.bengali_script)"},
{"utterance": "how tall is mount everest", "targetFormula": "(!fb:geography.mountain.elevation fb:en.mount_everest)"},
{"utterance": "who are the hosts of car talk", "targetFormula": "(!fb:broadcast.content.artist fb:en.car_talk)"},
{"utterance": "where was henry iii buried", "targetFormula": "(!fb:people.deceased_person.place_of_burial fb:en.henry_iii_of_england)"},
{"utterance": "who manufactures the t-38 talon", "targetFormula": "(!fb:aviation.aircraft_model.manufacturer fb:en.t-38_talon)"},
{"utterance": "what award does the royal institute of british architects present", "targetFormula": "(!fb:award.award_presenting_organization.awards_presented fb:en.royal_institute_of_british_architects)"},
{"utterance": "how many orders did queen victoria found", "targetFormula": "(count (!fb:royalty.chivalric_order_founder.orders_founded fb:en.victoria_of_the_united_kingdom))"},
{"utterance": "what was robert oppenheimer's role in the manhattan project", "targetFormula": "(!fb:projects.project_participation.role (and ((lambda x (fb:projects.project_participation.project (var x))) fb:en.manhattan_project) ((lambda x (fb:projects.project_participation.participant (var x))) fb:en.robert_oppenheimer)))"},
{"utterance": "who are the players currently on the los angeles dodgers", "targetFormula": "(!fb:baseball.baseball_roster_position.player ((lambda x (fb:baseball.baseball_roster_position.team (var x))) fb:en.los_angeles_dodgers))"},
{"utterance": "what religion did martin luther found", "targetFormula": "(!fb:religion.founding_figure.religion_founded fb:en.martin_luther)"},
{"utterance": "what character was born in gotham city", "targetFormula": "(!fb:fictional_universe.fictional_setting.fictional_characters_born_here fb:en.gotham_city)"},
{"utterance": "how much protein is in quinoa", "targetFormula": "(!fb:food.nutrition_fact.quantity (and ((lambda x (fb:food.nutrition_fact.food (var x))) fb:en.quinoa) ((lambda x (fb:food.nutrition_fact.nutrient (var x))) fb:en.protein)))"},
{"utterance": "when was the movie big daddy released", "targetFormula": "(!fb:film.film.initial_release_date fb:en.big_daddy)"},
{"utterance": "what city hosted the 1948 summer olympics", "targetFormula": "(!fb:olympics.olympic_games.host_city fb:en.1948_summer_olympics)"},
{"utterance": "how many beers are there on freebase", "targetFormula": "(!fb:freebase.type_profile.instance_count fb:food.beer)"},
{"utterance": "when was target founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.target_corporation)"},
{"utterance": "how many countries use uk money", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.uk))"},
{"utterance": "what are columbia university's colors", "targetFormula": "(!fb:education.educational_institution.colors fb:en.columbia_university)"},
{"utterance": "what instrument did lightnin' hopkins play", "targetFormula": "(!fb:music.group_member.instruments_played fb:en.lightnin_hopkins)"},
{"utterance": "what are the practices of sikhism", "targetFormula": "(!fb:religion.religion.practices fb:en.sikhism)"},
{"utterance": "how many people blog on beyond robson", "targetFormula": "(count (!fb:internet.blog.blogger fb:en.beyond_robson))"},
{"utterance": "how many industries does walmart consist of", "targetFormula": "(count (!fb:business.business_operation.industry fb:en.wal-mart))"},
{"utterance": "what is the population in iowa", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.iowa))"},
{"utterance": "where does the market-frankford line stop", "targetFormula": "(!fb:metropolitan_transit.transit_line.stops fb:en.market-frankford_line)"},
{"utterance": "what are the subclasses of the boson class", "targetFormula": "(!fb:physics.particle_family.subclasses fb:en.boson)"},
{"utterance": "how long is the golden gate bridge", "targetFormula": "(!fb:transportation.bridge.total_length fb:en.golden_gate_bridge)"},
{"utterance": "how many boats does the royal navy own", "targetFormula": "(count (!fb:boats.ship_ownership.ship ((lambda x (fb:boats.ship_ownership.owner (var x))) fb:en.navy_dept)))"},
{"utterance": "where was luke skywalker born", "targetFormula": "(!fb:fictional_universe.fictional_character.place_of_birth fb:en.luke_skywalker)"},
{"utterance": "who was the story of charlie_s angels by", "targetFormula": "(!fb:film.film.written_by fb:en.charlies_angels)"},
{"utterance": "what was the manhattan project about", "targetFormula": "(!fb:projects.project.project_focus fb:en.manhattan_project)"},
{"utterance": "what party did grover cleveland belong to", "targetFormula": "(!fb:government.political_party_tenure.party ((lambda x (fb:government.political_party_tenure.politician (var x))) fb:en.grover_cleveland))"},
{"utterance": "what is the passenger capacity of a cessna citation x", "targetFormula": "(!fb:measurement_unit.integer_range.high_value (!fb:aviation.aircraft_model.passengers fb:en.cessna_citation_x))"},
{"utterance": "when was panasonic corporation founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.matsushita_electric_industrial_co)"},
{"utterance": "who got the gold medal in men_s singles tennis at the 1896 summer olympics", "targetFormula": "(!fb:olympics.olympic_medal_honor.medalist (and ( (lambda x (fb:olympics.olympic_medal_honor.event (var x))) fb:en.tennis_at_the_1896_summer_olympics_mens_singles) ((lambda x (fb:olympics.olympic_medal_honor.medal (var x))) fb:en.gold_medal)))"},
{"utterance": "in what year did harry potter and the goblet of fire win the hugo award for best novel", "targetFormula": "(!fb:award.award_honor.year (and ((lambda x (fb:award.award_honor.award (var x))) fb:en.hugo_award_for_best_novel) ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.harry_potter_and_the_goblet_of_fire)))"},
{"utterance": "how many dinosaur species are there in north america", "targetFormula": "(count (!fb:base.dinosaur.dinosaur_location.dinosaur_s fb:en.north_america))"},
{"utterance": "what is the parent disease of a stroke", "targetFormula": "(!fb:medicine.disease.parent_disease fb:en.stroke)"},
{"utterance": "who directed 13 going on 30", "targetFormula": "(!fb:film.film.directed_by fb:en.13_going_on_30)"},
{"utterance": "what team does dennis bergkamp play for", "targetFormula": "(!fb:soccer.football_roster_position.team ((lambda x (fb:soccer.football_roster_position.player (var x))) fb:en.dennis_bergkamp))"},
{"utterance": "in how many military conflicts did mussolini command", "targetFormula": "(count (!fb:military.military_command.military_conflict ((lambda x (fb:military.military_command.military_commander (var x))) fb:en.benito_mussolini)))"},
{"utterance": "what structures did frank lloyd wright design", "targetFormula": "(!fb:architecture.architect.structures_designed fb:en.frank_lloyd_wright)"},
{"utterance": "in what time zone is illinois", "targetFormula": "(!fb:location.location.time_zones fb:en.illinois)"},
{"utterance": "who are the main cast members of sesame street", "targetFormula": "(!fb:tv.regular_tv_appearance.actor ((lambda x (fb:tv.regular_tv_appearance.series (var x))) fb:en.sesame_street))"},
{"utterance": "who are the characters in to kill a mockingbird", "targetFormula": "(!fb:book.book.characters fb:en.to_kill_a_mockingbird)"},
{"utterance": "who are the students of bruce lee", "targetFormula": "(!fb:martial_arts.martial_artist.martial_arts_students fb:en.bruce_lee)"},
{"utterance": "how tall was seabiscuit", "targetFormula": "(!fb:biology.organism.height_meters fb:en.seabiscuit)"},
{"utterance": "who won muhammad ali vs. joe frazier ii", "targetFormula": "(!fb:boxing.match_boxer_relationship.boxer (and ((lambda x (fb:boxing.match_boxer_relationship.winner_won (var x))) true) ((lambda x (fb:boxing.match_boxer_relationship.match (var x))) fb:en.ali-frazier_ii)))"},
{"utterance": "what is the engine in a 2010 ferrari california", "targetFormula": "(!fb:automotive.trim_level.engine fb:m.0h33xbs)"},
{"utterance": "how many other names is panasonic corporation known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.matsushita_electric_industrial_co))"},
{"utterance": "what genre did lil hardin armstrong represent", "targetFormula": "(!fb:music.artist.genre fb:en.lil_hardin_armstrong)"},
{"utterance": "what is currency code for the japanese yen", "targetFormula": "(!fb:finance.currency.currency_code fb:en.japanese_yen)"},
{"utterance": "what was the fawlty towers episode after the psychiatrist", "targetFormula": "(!fb:tv.tv_series_episode.next_episode fb:en.the_psychiatrist)"},
{"utterance": "who is the president of gap, inc.", "targetFormula": "(!fb:business.employment_tenure.person (and ((lambda x (fb:business.employment_tenure.company (var x))) fb:en.gap_inc) ((lambda x (fb:business.employment_tenure.title (var x))) fb:en.president)))"},
{"utterance": "what chemical series is tungsten in", "targetFormula": "(!fb:chemistry.chemical_element.chemical_series fb:en.tungsten)"},
{"utterance": "how many works have been set on the moon", "targetFormula": "(count (!fb:fictional_universe.fictional_setting.works_set_here fb:en.moon))"},
{"utterance": "when was jcpenney founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.j_c_penney)"},
{"utterance": "what is the setting of the magician_s nephew", "targetFormula": "(!fb:fictional_universe.work_of_fiction.setting fb:en.the_magicians_nephew)"},
{"utterance": "who produces meet the press", "targetFormula": "(!fb:broadcast.content.producer fb:en.nbc_meet_the_press_video)"},
{"utterance": "when did jack albertson die", "targetFormula": "(!fb:people.deceased_person.date_of_death fb:en.jack_albertson)"},
{"utterance": "what tv programs has hugh laurie created", "targetFormula": "(fb:tv.tv_program.program_creator fb:en.hugh_laurie)"},
{"utterance": "how many breeds are in the sporting group of the american kennel club", "targetFormula": "(count (!fb:biology.breed_registration.breed (and ((lambda x (fb:biology.breed_registration.breed_group (var x))) fb:en.sporting_group) ((lambda x (fb:biology.breed_registration.registry (var x))) fb:en.american_kennel_club))))"},
{"utterance": "how many visitors does mammoth cave national park get in a year", "targetFormula": "(!fb:measurement_unit.dated_integer.number (!fb:protected_sites.protected_site.annual_visitors fb:en.mammoth_cave_national_park))"},
{"utterance": "how many animals are there at the london zoo", "targetFormula": "(!fb:zoos.zoo.num_animals fb:en.london_zoo)"},
{"utterance": "where is independence hall located", "targetFormula": "(!fb:location.location.containedby fb:en.independence_hall)"},
{"utterance": "when was mark mckinney born", "targetFormula": "(!fb:people.person.date_of_birth fb:en.mark_mckinney)"},
{"utterance": "when were ritz crackers introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.ritz_cracker))"},
{"utterance": "when was tostitos introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.tostitos))"},
{"utterance": "what are some periodicals on computer science", "targetFormula": "(!fb:book.periodical_subject.periodicals fb:en.computer_science)"},
{"utterance": "in what year did motorola have the most revenue", "targetFormula": "(!fb:measurement_unit.dated_money_value.valid_date ((lambda x (!fb:business.business_operation.revenue (var x))) fb:en.motorola))"},
{"utterance": "how many other names is prostate cancer known as", "targetFormula": "(count (!fb:common.topic.alias fb:en.prostate_cancer))"},
{"utterance": "who are the creators of teenage mutant ninja turtles", "targetFormula": "(!fb:tv.tv_program.program_creator fb:m.053x52)"},
{"utterance": "who proposed the fifth amendment", "targetFormula": "(!fb:law.constitutional_amendment.proposed_by fb:en.fifth_amendment_to_the_united_states_constitution)"},
{"utterance": "what programming languages were used for aol instant messenger", "targetFormula": "(!fb:computer.software.languages_used fb:en.aol_instant_messenger)"},
{"utterance": "how many awards did sesame street win", "targetFormula": "(count (!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.sesame_street)))"},
{"utterance": "in what division are the st. louis cardinals", "targetFormula": "(!fb:baseball.baseball_team.division fb:en.st_louis_cardinals)"},
{"utterance": "what are the dna codons of glutamine", "targetFormula": "(!fb:biology.amino_acid.codons fb:en.glutamine)"},
{"utterance": "how many us supreme court cases have there been", "targetFormula": "(count (!fb:law.court.legal_cases fb:en.supreme_court_of_the_united_states))"},
{"utterance": "who is currently number 6 on the washington wizards", "targetFormula": "(!fb:basketball.basketball_roster_position.player (and ((lambda x (fb:basketball.basketball_roster_position.team (var x))) fb:en.washington_wizards) ((lambda x (fb:basketball.basketball_roster_position.number (var x))) (number 6.0 fb:en.unitless))))"},
{"utterance": "what makes does ford have", "targetFormula": "(!fb:automotive.company.make_s fb:en.ford_motor_company)"},
{"utterance": "what are some films on the apollo 11 mission", "targetFormula": "(fb:film.film.subjects fb:en.apollo_11)"},
{"utterance": "what are some books about art deco", "targetFormula": "(!fb:book.book_subject.works fb:en.art_deco)"},
{"utterance": "what is the official language of mauritania", "targetFormula": "(!fb:location.country.official_language fb:en.mauritania)"},
{"utterance": "what cities does us interstate 5 pass", "targetFormula": "(!fb:transportation.road.major_cities fb:en.us_interstate_5)"},
{"utterance": "how many people collect automobiles", "targetFormula": "(count (!fb:interests.collection.collector ((lambda x (fb:interests.collection.category (var x))) fb:en.automobile)))"},
{"utterance": "how many parent types does igneous rock have", "targetFormula": "(count (!fb:geology.rock_type.parent_rock_type fb:en.igneous_rock))"},
{"utterance": "what is the piston configuration of a ford model t engine", "targetFormula": "(!fb:engineering.piston_engine.piston_configuration fb:en.ford_model_t_engine)"},
{"utterance": "what digital cameras take an sdhc card", "targetFormula": "(!fb:digicams.camera_storage_type.compatible_cameras fb:en.sdhc_card)"},
{"utterance": "where is the uss alabama currently moored", "targetFormula": "(!fb:location.location.containedby fb:m.019zhn)"},
{"utterance": "what are the divisions of the western conference of the nhl", "targetFormula": "(!fb:ice_hockey.hockey_conference.divisions fb:en.western_conference)"},
{"utterance": "what were the highest wind speeds in hurricane katrina", "targetFormula": "(!fb:meteorology.tropical_cyclone.highest_winds fb:en.hurricane_katrina)"},
{"utterance": "how long overall is the tahina", "targetFormula": "(!fb:boats.ship.length_overall fb:m.0b_h31m)"},
{"utterance": "when was the mark vii monorail introduced", "targetFormula": "(!fb:rail.locomotive_class.introduced fb:en.mark_vii_monorail)"},
{"utterance": "where is walden pond located", "targetFormula": "(!fb:location.location.containedby fb:en.walden_pond)"},
{"utterance": "what diseases is fatigue a symptom of", "targetFormula": "(fb:medicine.disease.symptoms fb:en.fatigue)"},
{"utterance": "what rock formations formed in the barremian period", "targetFormula": "(fb:geology.geological_formation.formed_during_period fb:en.barremian)"},
{"utterance": "what is jcpenney 's operating income", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:business.business_operation.operating_income (var x))) fb:en.j_c_penney))"},
{"utterance": "what types of judaism are there", "targetFormula": "(!fb:religion.religion.includes fb:en.judaism)"},
{"utterance": "how many film performances did ron glass do", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.ron_glass)))"},
{"utterance": "where was the uss constitution built", "targetFormula": "(!fb:boats.ship.place_built fb:en.uss_constitution)"},
{"utterance": "what treatments are there for leukemia", "targetFormula": "(!fb:medicine.disease.treatments fb:en.leukemia)"},
{"utterance": "what kind of website is bing", "targetFormula": "(!fb:internet.website.category fb:en.windows_live_search)"},
{"utterance": "when did the standard school broadcast start production", "targetFormula": "(!fb:broadcast.content.production_start fb:en.the_standard_school_broadcast)"},
{"utterance": "what are the rides at six flags america", "targetFormula": "(!fb:amusement_parks.park.rides fb:en.six_flags_america)"},
{"utterance": "where did omarion 's musical career begin", "targetFormula": "(!fb:music.artist.origin fb:en.omarion_grandberry)"},
{"utterance": "how many tracks has the band recorded", "targetFormula": "(count (!fb:music.artist.track fb:en.the_band))"},
{"utterance": "what articles are in the july 1967 issue of frontier times", "targetFormula": "(!fb:book.contents.work ((lambda x (fb:book.contents.publication (var x))) fb:en.frontier_times_1967_july))"},
{"utterance": "what species is spock", "targetFormula": "(!fb:fictional_universe.fictional_character.species fb:en.spock)"},
{"utterance": "how many knights of the garter did the order of the garter have", "targetFormula": "(count (!fb:royalty.chivalric_order_membership.recipient (and ((lambda x (fb:royalty.chivalric_order_membership.order (var x))) fb:en.order_of_the_garter) ((lambda x (fb:royalty.chivalric_order_membership.title (var x))) fb:en.knight_of_the_garter))))"},
{"utterance": "when did charlie_s angels come out", "targetFormula": "(!fb:film.film.initial_release_date fb:en.charlies_angels)"},
{"utterance": "at what age to kids usually start second grade", "targetFormula": "(!fb:education.grade_level.typical_age_minimum fb:en.second_grade)"},
{"utterance": "when was yosemite national park designated as a protected place", "targetFormula": "(!fb:protected_sites.protected_site.date_established fb:en.yosemite_national_park)"},
{"utterance": "what substancs has robert downey jr abused", "targetFormula": "(!fb:celebrities.substance_abuse_problem.substance ((lambda x (fb:celebrities.substance_abuse_problem.celebrity (var x))) fb:en.robert_downey_jr))"},
{"utterance": "what did elizabeth ii collect", "targetFormula": "(!fb:interests.collection.category ((lambda x (fb:interests.collection.collector (var x))) fb:en.elizabeth_ii_of_the_united_kingdom))"},
{"utterance": "who owned doritos", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.doritos))"},
{"utterance": "how many sub-types of metamorphic rock are there", "targetFormula": "(count (!fb:geology.rock_type.sub_types fb:en.metamorphic_rock))"},
{"utterance": "what species is the doctor", "targetFormula": "(!fb:fictional_universe.fictional_character.species fb:en.the_doctor)"},
{"utterance": "when did chips ahoy debut", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.chips_ahoy))"},
{"utterance": "what is the price of a 2012 jeep wrangler sport", "targetFormula": "(!fb:measurement_unit.money_value.amount (!fb:automotive.trim_level.msrp fb:m.0hgwnr4))"},
{"utterance": "how many nominations did nutty professor have", "targetFormula": "(count (!fb:award.award_nomination.award ((lambda x (fb:award.award_nomination.nominated_for (var x))) fb:en.the_nutty_professor_1996)))"},
{"utterance": "what companies produce smartphones", "targetFormula": "(!fb:business.competitive_space_mediator.company ((lambda x (fb:business.competitive_space_mediator.space (var x))) fb:en.smartphone))"},
{"utterance": "how many french operas are there", "targetFormula": "(count (fb:opera.opera.language fb:en.french))"},
{"utterance": "what is the population of the people on the earth", "targetFormula": "(!fb:measurement_unit.dated_integer.number ((lambda x (!fb:location.statistical_region.population (var x))) fb:en.earth))"},
{"utterance": "what opera did jonathan miller direct", "targetFormula": "(!fb:opera.opera_director.operas_directed fb:en.jonathan_miller)"},
{"utterance": "what are some types of hats", "targetFormula": "(!fb:fashion.garment.more_specialized_forms fb:en.hat)"},
{"utterance": "when was scope introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:m.02r3cjp))"},
{"utterance": "what award did jack albertson win in 1968", "targetFormula": "(!fb:award.award_honor.award (and ((lambda x (fb:award.award_honor.year (var x))) (date 1968 -1 -1)) ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.jack_albertson)))"},
{"utterance": "who makes up tomkat", "targetFormula": "(!fb:celebrities.supercouple.partners fb:en.tomkat)"},
{"utterance": "what area of disneyland is space mountain in", "targetFormula": "(!fb:amusement_parks.ride.area fb:en.space_mountain)"},
{"utterance": "who was the music by in titanic", "targetFormula": "(!fb:film.film.music fb:en.titanic_special_edition_dvd)"},
{"utterance": "what military branches use the f-5 freedom fighter", "targetFormula": "(!fb:aviation.aircraft_ownership_count.aircraft_owner ((lambda x (fb:aviation.aircraft_ownership_count.aircraft_model (var x))) fb:en.f-5_freedom_fighter))"},
{"utterance": "what are the causes of syphilis", "targetFormula": "(!fb:medicine.disease.causes fb:en.syphillis)"},
{"utterance": "what genre of music is b12", "targetFormula": "(!fb:music.artist.genre fb:en.b12)"},
{"utterance": "who designs for bill blass limited", "targetFormula": "(!fb:fashion.designer_label_association.designer ((lambda x (fb:fashion.designer_label_association.label (var x))) fb:en.bill_blass_limited))"},
{"utterance": "when did john f kennedy die", "targetFormula": "(!fb:people.deceased_person.date_of_death fb:en.john_f_kennedy)"},
{"utterance": "who was vice president to woodrow wilson", "targetFormula": "(!fb:government.us_president.vice_president fb:en.woodrow_wilson)"},
{"utterance": "when was borders founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.borders_group)"},
{"utterance": "who fought in the the battle of the champions", "targetFormula": "(!fb:boxing.match_boxer_relationship.boxer ((lambda x (fb:boxing.match_boxer_relationship.match (var x))) fb:en.the_battle_of_the_champions))"},
{"utterance": "who was the cohost in 1993 on cbs evening news", "targetFormula": "(!fb:tv.tv_regular_personal_appearance.person (and (and ((lambda x (fb:tv.tv_regular_personal_appearance.program (var x))) fb:en.cbs_evening_news) ((lambda x (fb:tv.tv_regular_personal_appearance.appearance_type (var x))) fb:en.co_host)) ((lambda x (fb:tv.tv_regular_personal_appearance.from (var x))) (date 1993 -1 -1))))"},
{"utterance": "for what albums did barack obama win a grammy award for best spoken word album", "targetFormula": "(!fb:award.award_honor.honored_for (and ((lambda x (fb:award.award_honor.award (var x))) fb:en.grammy_award_for_best_spoken_word_album) ((lambda x (fb:award.award_honor.award_winner (var x))) fb:en.barack_obama)))"},
{"utterance": "how many other names is omarion known by", "targetFormula": "(count (!fb:common.topic.alias fb:en.omarion_grandberry))"},
{"utterance": "what percent alcohol is a g.h. mumm cordon rouge brut", "targetFormula": "(!fb:wine.wine.percentage_alcohol fb:en.g_h_mumm_cordon_rouge_brut)"},
{"utterance": "how many countries use the indian rupee", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.indian_rupee))"},
{"utterance": "when was the uss croaker launched", "targetFormula": "(!fb:boats.ship.launched fb:en.uss_croaker)"},
{"utterance": "who is the producer for 13 going on 30", "targetFormula": "(!fb:film.film.produced_by fb:en.13_going_on_30)"},
{"utterance": "what is the australian dollar code", "targetFormula": "(!fb:finance.currency.currency_code fb:en.australian_dollar)"},
{"utterance": "who composed the song ship of fools", "targetFormula": "(!fb:music.composition.composer fb:m.02knmd5)"},
{"utterance": "what are some adaptations of the wonderful wizard of oz", "targetFormula": "(!fb:media_common.adapted_work.adaptations fb:en.the_wonderful_wizard_of_oz)"},
{"utterance": "what generation of matter is a down quark", "targetFormula": "(!fb:physics.particle.generation fb:en.down_quark)"},
{"utterance": "what is the max speed of kingda ka", "targetFormula": "(!fb:amusement_parks.ride.max_speed fb:en.kingda_ka)"},
{"utterance": "what is ron glass 's place of birth", "targetFormula": "(!fb:people.person.place_of_birth fb:en.ron_glass)"},
{"utterance": "who was rolling stones' greatest guitarist of all time in 2003", "targetFormula": "(!fb:award.ranking.item ( (lambda x (fb:award.ranking.list (var x))) fb:en.rolling_stones_100_greatest_guitarists_of_all_time))"},
{"utterance": "what religion was pablo picasso", "targetFormula": "(!fb:people.person.religion fb:en.pablo_picasso)"},
{"utterance": "who is the owner of easy cheese", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.easy_cheese))"},
{"utterance": "which characters are playable in sonic rush", "targetFormula": "(!fb:cvg.game_performance.character ((lambda x (fb:cvg.game_performance.game (var x))) fb:en.sonic_rush))"},
{"utterance": "who created the runaways comic book series", "targetFormula": "(!fb:comic_books.comic_book_series.created_by fb:en.runaways)"},
{"utterance": "what textiles are made from cotton", "targetFormula": "(!fb:fashion.fiber.textiles_made_from_this_fiber fb:en.cotton)"},
{"utterance": "what is the theme song of sesame street", "targetFormula": "(!fb:tv.tv_program.theme_song fb:en.sesame_street)"},
{"utterance": "what movies won the golden globe award for best drama film", "targetFormula": "(!fb:award.award_honor.honored_for ( (lambda x (fb:award.award_honor.award (var x))) fb:en.golden_globe_award_for_best_motion_picture_-_drama))"},
{"utterance": "what are the causes of a heart attack", "targetFormula": "(!fb:medicine.disease.causes fb:en.heart_failure)"},
{"utterance": "how many main characters did charlie_s angels have", "targetFormula": "(count (!fb:film.performance.character ((lambda x (fb:film.performance.film (var x))) fb:en.charlies_angels)))"},
{"utterance": "what ship class is the uss cobia", "targetFormula": "(!fb:boats.ship.ship_class fb:en.uss_cobia)"},
{"utterance": "how many tv shows did jerry seinfeld produce", "targetFormula": "(count (!fb:tv.tv_producer_term.program ((lambda x (fb:tv.tv_producer_term.producer (var x))) fb:en.jerry_seinfeld)))"},
{"utterance": "what rock formations are made of quartzite", "targetFormula": "(!fb:geology.rock_type.formations fb:en.quartzite)"},
{"utterance": "what is rna binding", "targetFormula": "(!fb:biology.gene_ontology_group.description fb:en.rna_binding)"},
{"utterance": "when did the band massive attack form", "targetFormula": "(!fb:music.artist.active_start fb:en.massive_attack)"},
{"utterance": "when is the restoration of the rijksmuseum supposed to finish", "targetFormula": "(!fb:projects.project.planned_completion_date fb:m.0fq7hj3)"},
{"utterance": "who founded gsusa", "targetFormula": "(!fb:organization.organization.founders fb:en.girl_scouts_of_the_usa)"},
{"utterance": "how many products has mattel produced", "targetFormula": "(count (!fb:business.company_product_relationship.consumer_product (!fb:business.consumer_company.products fb:en.mattel)))"},
{"utterance": "when was techvibes started", "targetFormula": "(!fb:internet.blog.started fb:en.techvibes)"},
{"utterance": "who won the 1964 united states presidential election", "targetFormula": "(!fb:government.election.winner fb:en.united_states_presidential_election_1964)"},
{"utterance": "how many countries are within north america", "targetFormula": "(count (fb:location.location.containedby fb:en.north_america))"},
{"utterance": "how many films are there on antarctica", "targetFormula": "(count (fb:film.film.subjects fb:en.antarctica))"},
{"utterance": "when was the green party founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.green_party)"},
{"utterance": "when did the princetown railway close", "targetFormula": "(!fb:rail.railway.closed fb:en.princetown_railway)"},
{"utterance": "who are the characters in peanuts", "targetFormula": "(!fb:comic_strips.comic_strip.characters fb:en.peanuts)"},
{"utterance": "how much danish kroner is a us dollar worth", "targetFormula": "(!fb:finance.exchange_rate.target_of_exchange ((lambda x (fb:finance.exchange_rate.source_of_exchange (var x))) fb:en.danish_krone))"},
{"utterance": "what series is a wrinkle in time a part of", "targetFormula": "(!fb:book.written_work.part_of_series fb:en.a_wrinkle_in_time)"},
{"utterance": "how many soccer players are goalkeepers", "targetFormula": "(count (!fb:soccer.football_position.players fb:en.goalkeeper_football))"},
{"utterance": "what software has oracle designed", "targetFormula": "(!fb:computer.software_developer.software fb:en.oracle_corporation)"},
{"utterance": "how much time did alan shepard spend in space", "targetFormula": "(!fb:spaceflight.astronaut.space_time_minutes fb:en.alan_shepard)"},
{"utterance": "how many hockey players play center", "targetFormula": "(count (!fb:ice_hockey.hockey_position.players fb:m.02qvdc))"},
{"utterance": "who is wells fargo 's supplier", "targetFormula": "(!fb:business.customer.supplier fb:en.wells_fargo)"},
{"utterance": "who were the members of the soul stirrers", "targetFormula": "(!fb:music.group_membership.member ((lambda x (fb:music.group_membership.group (var x))) fb:en.the_soul_stirrers))"},
{"utterance": "where do the denver spurs play", "targetFormula": "(!fb:sports.sports_team.arena_stadium fb:en.denver_spurs)"},
{"utterance": "who won best movie of 2010", "targetFormula": "(!fb:award.award_honor.award_winner (and ((lambda x (fb:award.award_honor.award (var x))) fb:en.academy_award_for_best_picture) ((lambda x (fb:award.award_honor.year (var x))) (date 2010 -1 -1))))"},
{"utterance": "what hospitals specialize in cardiology", "targetFormula": "(!fb:medicine.medical_specialty.hospitals_with_this_specialty fb:en.cardiology)"},
{"utterance": "when was hank aaron inducted into the hall of fame", "targetFormula": "(!fb:baseball.baseball_player.hall_of_fame_induction fb:en.henry_aaron)"},
{"utterance": "what blog does robert scoble write", "targetFormula": "(!fb:internet.blogger.blog fb:en.robert_scoble)"},
{"utterance": "when was girl scouts of the usa founded", "targetFormula": "(!fb:organization.organization.date_founded fb:en.girl_scouts_of_the_usa)"},
{"utterance": "what is the atomic number of silicon", "targetFormula": "(!fb:chemistry.chemical_element.atomic_number fb:en.silicon)"},
{"utterance": "when did the renaissance begin", "targetFormula": "(!fb:time.event.start_date fb:en.renaissance)"},
{"utterance": "what year was danny devito born", "targetFormula": "(!fb:people.person.date_of_birth fb:en.danny_devito)"},
{"utterance": "how much did transformers cost to produce", "targetFormula": "(!fb:measurement_unit.dated_money_value.amount ((lambda x (!fb:film.film.estimated_budget (var x))) fb:en.transformers))"},
{"utterance": "where is the guardian circulated", "targetFormula": "(!fb:book.newspaper.circulation_areas fb:en.the_guardian)"},
{"utterance": "what type of beer is miller lite", "targetFormula": "(!fb:food.beer.beer_style fb:en.miller_lite)"},
{"utterance": "what is the temperature of polaris", "targetFormula": "(!fb:astronomy.star.temperature_k fb:m.0kjyrc7)"},
{"utterance": "what are dave grohl 's musical genres", "targetFormula": "(!fb:music.artist.genre fb:en.dave_grohl)"},
{"utterance": "what is currency code for the singapore dollar", "targetFormula": "(!fb:finance.currency.currency_code fb:en.singapore_dollar)"},
{"utterance": "what movies won ascap film and television music awards for most performed songs from motion pictures", "targetFormula": "(!fb:award.award_honor.honored_for ((lambda x (fb:award.award_honor.award (var x))) fb:m.04d215m))"},
{"utterance": "what is the scientific name for a horse", "targetFormula": "(!fb:biology.organism_classification.scientific_name fb:en.domesticated_horse)"},
{"utterance": "what artists are influenced by lead belly", "targetFormula": "(!fb:influence.influence_node.influenced fb:en.leadbelly)"},
{"utterance": "how many film productions did hayden panettiere appear in", "targetFormula": "(count (!fb:film.performance.film ((lambda x (fb:film.performance.actor (var x))) fb:en.hayden_panettiere)))"},
{"utterance": "what is currency code for the swedish krona", "targetFormula": "(!fb:finance.currency.currency_code fb:en.swedish_krona)"},
{"utterance": "who coaches the toronto maple leafs", "targetFormula": "(!fb:ice_hockey.hockey_team.coach fb:en.toronto_maple_leafs)"},
{"utterance": "who was titanic produced by", "targetFormula": "(!fb:film.film.produced_by fb:en.titanic_special_edition_dvd)"},
{"utterance": "how many topics are equivalent to city/town/village", "targetFormula": "(count (!fb:freebase.type_profile.equivalent_topic fb:location.citytown))"},
{"utterance": "what are some cajun dishes", "targetFormula": "(!fb:dining.cuisine.dishes fb:m.01v7z)"},
{"utterance": "who did the cover art for batman #477", "targetFormula": "(!fb:book.magazine_issue.cover_artist fb:en.batman_477)"},
{"utterance": "how many countries use euros", "targetFormula": "(count (!fb:finance.currency.countries_used fb:en.euro))"},
{"utterance": "how many storms were in the 2005 atlantic hurricane season", "targetFormula": "(!fb:meteorology.tropical_cyclone_season.total_storms fb:en.2005_atlantic_hurricane_season)"},
{"utterance": "what was the strongest storm in the 1992 atlantic hurricane season", "targetFormula": "(!fb:meteorology.tropical_cyclone_season.strongest_storm fb:en.1992_atlantic_hurricane_season)"},
{"utterance": "when did the apple i stop being sold", "targetFormula": "(!fb:computer.computer.discontinued fb:en.apple_i)"},
{"utterance": "who has held the title of wba world champion", "targetFormula": "(!fb:boxing.boxing_title_tenure.champion ((lambda x (fb:boxing.boxing_title_tenure.title (var x))) fb:m.0chgh2j))"},
{"utterance": "who studied martial arts under yip man", "targetFormula": "(!fb:martial_arts.martial_artist.martial_arts_students fb:en.yip_man)"},
{"utterance": "how many losses has sean miller had in his career", "targetFormula": "(!fb:basketball.basketball_coach.season_losses fb:en.sean_miller_1968)"},
{"utterance": "who wrote travels with my cello", "targetFormula": "(!fb:book.written_work.author fb:m.067y_k7)"},
{"utterance": "who were the parents of ferdinand ii", "targetFormula": "(!fb:people.person.parents fb:en.ferdinand_ii_holy_roman_emperor)"},
{"utterance": "who operates the silver star railway", "targetFormula": "(!fb:rail.railway_operator_relationship.operator ((lambda x (fb:rail.railway_operator_relationship.railway (var x))) fb:m.04rsv0))"},
{"utterance": "what labels does sarah jessica parker wear", "targetFormula": "(!fb:base.popstra.fashion_choice.designer ((lambda x (fb:base.popstra.fashion_choice.fashion_wearer (var x))) fb:en.sarah_jessica_parker))"},
{"utterance": "what awards has lost won", "targetFormula": "(!fb:award.award_honor.award ((lambda x (fb:award.award_honor.honored_for (var x))) fb:en.lost))"},
{"utterance": "what ingredients are in italian cuisine", "targetFormula": "(!fb:dining.cuisine.ingredients fb:m.09y2k2)"},
{"utterance": "what is the max speed of a collins class submarine", "targetFormula": "(!fb:boats.ship_class.max_speed_knots fb:en.collins_class_submarine)"},
{"utterance": "what country does the royal navy belong to", "targetFormula": "(!fb:military.armed_force.military_combatant fb:en.navy_dept)"},
{"utterance": "who played in the 1973 fa cup final", "targetFormula": "(!fb:soccer.football_match.teams fb:en.fa_cup_final_1973)"},
{"utterance": "how many teams did jim thorpe play for", "targetFormula": "(count (!fb:american_football.football_historical_roster_position.team ( (lambda x (fb:american_football.football_historical_roster_position.player (var x))) fb:en.jim_thorpe)))"},
{"utterance": "what are sacred sites in sunni islam", "targetFormula": "(!fb:religion.religion.sacred_sites fb:en.sunni_islam)"},
{"utterance": "who produced country time", "targetFormula": "(!fb:business.company_brand_relationship.company ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.country_time))"},
{"utterance": "what are some german beers", "targetFormula": "(!fb:food.beer_country_region.beers_from_here fb:en.germany)"},
{"utterance": "what is the currency code for the indonesian rupiah", "targetFormula": "(!fb:finance.currency.currency_code fb:en.indonesian_rupiah)"},
{"utterance": "what is sarah jessica parker's hobby", "targetFormula": "(!fb:interests.hobbyist.hobbies fb:en.sarah_jessica_parker)"},
{"utterance": "what fictional characters have attended oxford", "targetFormula": "(!fb:fictional_universe.school_in_fiction.students_graduates fb:en.university_of_oxford)"},
{"utterance": "how many people died in the boston molasses disaster", "targetFormula": "(!fb:event.disaster.fatalities fb:en.boston_molasses_disaster)"},
{"utterance": "what position does keyon dooling play", "targetFormula": "(!fb:basketball.basketball_player.position_s fb:en.keyon_dooling)"},
{"utterance": "how many oscar shows did frank sinatra host", "targetFormula": "(count (!fb:base.academyawards.host_of_oscar_show.oscar_show_hosted fb:en.frank_sinatra))"},
{"utterance": "where was the cross of gold speech presented", "targetFormula": "(!fb:event.speech_or_presentation.event ((lambda x (fb:event.speech_or_presentation.presented_work (var x))) fb:en.cross_of_gold_speech))"},
{"utterance": "when did hurricane andrew form", "targetFormula": "(!fb:meteorology.tropical_cyclone.formed fb:en.hurricane_andrew)"},
{"utterance": "who designed the rolls-royce welland", "targetFormula": "(!fb:engineering.engine.designer fb:en.rolls-royce_welland)"},
{"utterance": "how many cancer centers are comprehensive", "targetFormula": "(count (fb:medicine.cancer_center.comprehensive true))"},
{"utterance": "how many albums has fleet foxes released", "targetFormula": "(count (!fb:music.artist.album fb:en.fleet_foxes))"},
{"utterance": "what is the earth 's area", "targetFormula": "(!fb:location.location.area fb:en.earth)"},
{"utterance": "how many other names are there for the usa_s currency", "targetFormula": "(count (!fb:common.topic.alias fb:en.us))"},
{"utterance": "who has portrayed james bond in film", "targetFormula": "(!fb:film.performance.actor ((lambda x (fb:film.performance.character (var x))) fb:m.0clpml))"},
{"utterance": "what is the slogan of bmw", "targetFormula": "(!fb:organization.organization.slogan fb:en.bmw)"},
{"utterance": "when did robin williams stop abusing cocaine", "targetFormula": "(!fb:celebrities.substance_abuse_problem.end (and ((lambda x (fb:celebrities.substance_abuse_problem.celebrity (var x))) fb:en.robin_williams) ((lambda x (fb:celebrities.substance_abuse_problem.substance (var x))) fb:en.cocaine)))"},
{"utterance": "how tall is the you only live once image", "targetFormula": "(!fb:common.image.size fb:m.0kk92b)"},
{"utterance": "what is the voltage of a aa alkaline battery", "targetFormula": "(!fb:engineering.battery_size_cell_variation.voltage (and ((lambda x (fb:engineering.battery_size_cell_variation.cell_type (var x))) fb:en.alkaline_battery) ((lambda x (fb:engineering.battery_size_cell_variation.size (var x))) fb:en.aa_battery)))"},
{"utterance": "how many stages of breast cancer are there", "targetFormula": "(count (!fb:medicine.disease.stages fb:en.breast_cancer))"},
{"utterance": "what languages are spoken in firefly", "targetFormula": "(!fb:tv.tv_program.languages fb:m.014v3t)"},
{"utterance": "when was the frida kahlo exhibit at the philadelphia art museum", "targetFormula": "(!fb:exhibitions.exhibition_run.opened_on (and ((lambda x (fb:exhibitions.exhibition_run.exhibition (var x))) fb:m.04d14h7) ((lambda x (fb:exhibitions.exhibition_run.venue (var x))) fb:en.philadelphia_museum_of_art)))"},
{"utterance": "when was stove top stuffing introduced", "targetFormula": "(!fb:business.company_brand_relationship.from_date ((lambda x (fb:business.company_brand_relationship.brand (var x))) fb:en.stove_top_stuffing))"},
{"utterance": "how many units does the us army have", "targetFormula": "(count (!fb:military.armed_force.units fb:en.u_army))"},
{"utterance": "who was the nutty professor produced by", "targetFormula": "(!fb:film.film.produced_by fb:en.the_nutty_professor_1996)"},
{"utterance": "how many documentaries are there on netflix", "targetFormula": "(count (!fb:media_common.netflix_genre.titles fb:en.documentary))"},
{"utterance": "who are the current senators of colorado", "targetFormula": "(!fb:government.government_position_held.office_holder (and ( (lambda x (fb:government.government_position_held.office_position_or_title (var x))) fb:en.united_states_senator) ((lambda x (fb:government.government_position_held.district_represented (var x))) fb:en.colorado)))"},
{"utterance": "how many songs are there for guitar hero: aerosmith", "targetFormula": "(count (!fb:cvg.musical_game_song_relationship.song ((lambda x (fb:cvg.musical_game_song_relationship.game (var x))) fb:en.guitar_hero_aerosmith)))"},
{"utterance": "what is lamarcus aldridge's number", "targetFormula": "(!fb:basketball.basketball_roster_position.number ((lambda x (fb:basketball.basketball_roster_position.player (var x))) fb:en.lamarcus_aldridge))"},
{"utterance": "what are the sub-types of coal", "targetFormula": "(!fb:geology.rock_type.sub_types fb:en.coal)"},
{"utterance": "how many books are there written on computer programming", "targetFormula": "(count (!fb:media_common.literary_genre.books_in_this_genre fb:en.computer_programming))"},
{"utterance": "where did the rolling stones 2009 concert tour take place", "targetFormula": "(!fb:time.event.locations fb:en.the_rolling_stones_2009_concert_tour)"},
{"utterance": "what is the electric charge of a proton", "targetFormula": "(!fb:physics.particle.electric_charge fb:en.proton)"},
{"utterance": "when was macintosh introduced", "targetFormula": "(!fb:computer.computer.introduced fb:en.macintosh)"},
{"utterance": "how many works did mozart dedicate to joseph haydn", "targetFormula": "(count (!fb:media_common.dedication.work_dedicated (and ((lambda x (fb:media_common.dedication.dedicated_to (var x))) fb:en.joseph_haydn) ((lambda x (fb:media_common.dedication.dedicated_by (var x))) fb:en.wolfgang_amadeus_mozart))))"},
{"utterance": "what is the subject of 13 going on 30", "targetFormula": "(!fb:film.film.subjects fb:en.13_going_on_30)"},
{"utterance": "who has influenced stephen fry", "targetFormula": "(!fb:influence.influence_node.influenced_by fb:en.stephen_fry)"},
{"utterance": "who is the costume designer for bruce almighty", "targetFormula": "(!fb:film.film.costume_design_by fb:en.bruce_almighty)"},
{"utterance": "who wrote the episode straight and true of the wire", "targetFormula": "(!fb:tv.tv_series_episode.writer fb:en.straight_and_true)"}
]

6
freebase/data/tutorial-freebase.grammar Normal file
View File

@ -0,0 +1,6 @@
(rule $Unary ($PHRASE) (SimpleLexiconFn (type fb:type.any)))
(rule $Binary ($PHRASE) (SimpleLexiconFn (type (-> fb:type.any fb:type.any))))
(rule $Set ($Unary) (IdentityFn))
(rule $Set ($Unary $Set) (MergeFn and))
(rule $Set ($Binary $Set) (JoinFn forward))
(rule $ROOT ($Set) (IdentityFn))

2
data/tutorial.ttl → freebase/data/tutorial.ttl
View File

@ -17,5 +17,5 @@ fb:en.mount_whitney fb:type.object.name "Mount Whitney"@en.
fb:en.california fb:type.object.name "California"@en.
fb:en.seattle fb:location.location.area "369.2"^^xsd:double.
fb:en.san_francisco fb:location.location.area "600.6"^^xsd:double.
fb:en.san_francisco fb:location.location.area "600.6"@en.
fb:en.los_angeles fb:location.location.area "1301.97"^^xsd:double.

0
data/unittest-learn-ccg.grammar → freebase/data/unittest-learn-ccg.grammar
View File

9
freebase/data/unittest-learn.examples Normal file
View File

@ -0,0 +1,9 @@
(example
(utterance "American")
(targetFormula (fb:people.person.nationality fb:en.united_states))
)
(example
(utterance "American born in Paris")
(targetFormula (and (fb:people.person.nationality fb:en.united_states) (fb:people.person.place_of_birth fb:en.paris)))
)

0
data/unittest-learn.grammar → freebase/data/unittest-learn.grammar
View File

11
scripts/extract-freebase-schema.rb → freebase/scripts/extract-freebase-schema.rb
View File

@ -3,7 +3,7 @@
require 'open-uri'
if ARGV.size != 2 then
puts "Usage: <SPARQL endpoint (e.g., http://jackson:3090/sparql)> <schema.ttl file>"
puts "Usage: <SPARQL endpoint (e.g., http://jonsson:3093/sparql)> <schema.ttl file>"
exit 1
end
$endpoint, $outPath = ARGV
@ -11,6 +11,7 @@ $endpoint, $outPath = ARGV
# Return set of raw lines from the query (probably don't want to use this function directly)
def makeQuery(query)
queryUrl = URI::encode("PREFIX fb: <http://rdf.freebase.com/ns/> #{query}")
#puts query
`curl -s '#{$endpoint}?query=#{queryUrl}'`.split(/\n/)
end
@ -90,6 +91,7 @@ end
def putsTriple(out, x, p, y)
out.puts [x, p, y].join("\t") + '.'
out.flush
end
def extractAll
@ -156,9 +158,16 @@ def extractAll
'fb:freebase.type_hints.mediator',
'fb:freebase.type_hints.included_types',
nil].compact.each { |p|
unique = (p == 'fb:type.property.schema') || (p == 'fb:type.property.expected_type')
items = {}
puts "Extracting #{p}"
getEntries("select ?x ?y { ?x #{p} ?y }") { |x,y|
next unless hasSupport[x]
if unique and items[x]
puts "ERROR: #{x} #{p} #{items[x]} already exists, trying to add #{y}, skipping..."
next
end
items[x] = y
putsTriple(out, x, p, y)
}
}

210
freebase/scripts/fbshell.rb Executable file
View File

@ -0,0 +1,210 @@
#!/usr/bin/ruby
require 'open-uri'
require 'json'
require 'nokogiri'
require 'socket'
<<EOF
A shell for exploring the Freebase graph.
Uses the Freebase Search API to lookup entities.
Uses the cache server to map from mid to canonical ids.
Uses SPARQL queries to lookup graph edges.
At each point in time we have a state which yields a list of new states.
Here are the possible types of states
- State: entity (example: fb:en.barack_obama)
List: list of (entity, property) states.
- State: (entity, property) (example: fb:en.barack_obama fb:people.person.children)
List: list of entity values.
- State: query string (example: obama)
List: list of entity matches from Freebase Search.
- State: (entity, target) (example: fb:en.barack_obama [fb:en.michelle_obama])
List: list of (entity, property) that lead to the target.
- State: (entity, property, target) (example: fb:en.barack_obama fb:en.profession [fb:en.bill_clinton])
List: list of entity values that match from both entity and target from property.
TODO
Commands:
obama search
--michelle obama find paths to target
3 push an element
-3 show an element but don't push it
.. pop the stack
! compute listing for all items in list
EOF
$mid2id = TCPSocket.new('localhost', 4000)
def call(command)
$mid2id.puts command
result = $mid2id.gets.strip
return nil if result == '__NULL__'
return result
end
call("open\t/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map")
# Explore the Freebase graph
def fbsearch(query)
return [{'id' => query}] if query =~ /^fb:/ # Specify ID verbatim
raw = open('https://www.googleapis.com/freebase/v1/search?query=' + URI::encode(query)).read
json = JSON::parse(raw)
results = json['result']
results.map { |r|
mid = r['mid'].gsub!(/^\//, 'fb:').sub!(/\//, '.') # /m/02mjmr => fb:m.02mjmr
id = r['id'] = call("get\t" + mid) # Canonicalize the ids to be consistent with our Freebase index
if not id
puts "Skipping entry with no id: #{r.inspect}"
nil
else
r
end
}.compact
end
def sparql(query)
query = 'PREFIX fb: <http://rdf.freebase.com/ns/> ' + query + ' LIMIT 1000'
encodedQuery = URI::encode(query).gsub(/\+/, '%2b')
queryUrl = "http://localhost:3093/sparql?query=#{encodedQuery}"
raw = open(queryUrl).read
raw.sub!(/<sparql.*>/, '<sparql>') # Remove namespace
xml = Nokogiri::XML(raw)
def normalize(s)
s ? s.content.sub(/^http:\/\/rdf.freebase.com\/ns\//, 'fb:') : nil
end
xml.xpath('//result')
end
def get_properties(entity)
query = "SELECT DISTINCT ?property ?property_name { #{entity} ?property ?value . ?property fb:type.object.name ?property_name }"
sparql(query).map { |r|
property = normalize(r.xpath('binding[@name="property"]/uri')[0])
property_name = normalize(r.xpath('binding[@name="property_name"]/literal')[0])
{'id' => entity, 'property' => property, 'property_name' => property_name}
}
end
def get_values(entity, property)
query = "SELECT DISTINCT ?value ?value_name { #{entity} #{property} ?value . OPTIONAL { ?value fb:type.object.name ?value_name } }"
sparql(query).map { |r|
value = normalize(r.xpath('binding[@name="value"]/uri')[0])
value = normalize(r.xpath('binding[@name="value"]/literal')[0]) if not value
value_name = normalize(r.xpath('binding[@name="value_name"]/literal')[0])
{'id' => value, 'name' => value_name}
}
end
def find_paths(item1, items2)
id1 = item1['id']
id2 = items2[0]['id'] # TODO: expand
#p [id1, id2]
query = "SELECT DISTINCT ?property ?property_name { #{id1} ?property #{id2} . ?property fb:type.object.name ?property_name }"
one_hop = sparql(query).map { |r|
property = normalize(r.xpath('binding[@name="property"]/uri')[0])
property_name = normalize(r.xpath('binding[@name="property_name"]/literal')[0])
{'id' => id1, 'property' => property, 'property_name' => property_name}
}
query = "SELECT DISTINCT ?property1 ?property_name1 ?property2 ?property_name2 { #{id1} ?property1 ?x . ?x ?property2 #{id2} . ?property1 fb:type.object.name ?property_name1 . ?property2 fb:type.object.name ?property_name2 }"
two_hops = sparql(query).map { |r|
property1 = normalize(r.xpath('binding[@name="property1"]/uri')[0])
property_name1 = normalize(r.xpath('binding[@name="property_name1"]/literal')[0])
property2 = normalize(r.xpath('binding[@name="property2"]/uri')[0])
property_name2 = normalize(r.xpath('binding[@name="property_name2"]/literal')[0])
{'id' => id1, 'property' => property1 + '/' + property2, 'property_name' => property_name1 + ' / ' + property_name2}
}
one_hop + two_hops
end
stack = [{'id' => 'fb:en.barack_obama'}]
#stack = [{'id' => 'fb:en.tom_cruise'}]
compute_list = lambda { |item|
if item['list']
# Done
elsif item['target']
item['list'] = find_paths(item, item['target']['list'])
elsif item['query']
item['list'] = fbsearch(item['query'])
elsif item['property']
item['list'] = get_values(item['id'], item['property'])
else
item['list'] = get_properties(item['id'])
end
}
push_item = lambda { |item|
compute_list.call(item)
stack << item
}
def render_values(values)
lim = 3
s = values[0...lim].map{|x| [x['id'], x['name']].compact.join("_")}.join(' | ')
s += " (#{values.size})" if values.size > lim
s
end
print_list = lambda { |filter|
# Print list of items for the top
item = stack[-1]
compute_list.call(item)
item['list'].each_with_index { |r,i|
s = [i, r['property'] || r['id'], r['property_name'], r['name']].join("\t")
s += "\t" + render_values(r['list']) if r['list']
puts s if (not filter) || s =~ /#{filter}/
}
}
while true
item = stack[-1]
print [item['query'], item['id'], item['property'], item['target'] && item['target']['list'][0]['id']].compact.join(' ') + '> '
line = gets
break if line == nil
line.chomp!
if line == ''
# List
print_list.call(nil)
elsif line =~ /^\/(.+)$/
# List with filter
print_list.call($1)
elsif line == '..'
# Pop the stack
stack.pop if stack.size > 1
print_list.call(nil)
elsif line == '!'
# call list on all children
compute_list.call(item)
item['list'].each { |r|
compute_list.call(r)
}
print_list.call(nil)
elsif line =~ /^(-)?(\d+)$/
# Go to a particular selection of the list
stay = $1
i = Integer($2)
compute_list.call(item)
if i < 0 || i >= item['list'].size
puts "Out of range (#{item['list'].size} items)"
else
push_item.call(item['list'][i])
print_list.call(nil)
stack.pop if stay
end
elsif line =~ /^--(.+)$/
# Find path to target
target = {'query' => $1}
compute_list.call(target)
if target['list'].size == 0
puts 'No matches'
else
push_item.call({'id' => item['id'], 'target' => target})
print_list.call(nil)
end
else
# Free form search form entities
push_item.call({'query' => line})
print_list.call(nil)
end
end
$mid2id.close

37
scripts/virtuoso → freebase/scripts/virtuoso
View File

@ -2,7 +2,7 @@
# This script provides a convenient wrapper for the Virtuoso SPARQL server.
$virtuosoPath = File.dirname($0)+"/../virtuoso-opensource"
$virtuosoPath = File.dirname($0)+"/../../virtuoso-opensource"
if not File.exists?($virtuosoPath)
puts "#{$virtuosoPath} does not exist"
exit 1
@ -27,8 +27,8 @@ def run(command)
end
def start
if ARGV.size != 2
puts "Usage: <db directory> <port>"
if ARGV.size < 2
puts "Usage: <db directory> <port> [memory (MB)]"
exit 1
end
dbPath = ARGV.shift
@ -37,7 +37,11 @@ def start
Dir.mkdir(dbPath) if not File.exists?(dbPath)
# Recommended: 70% of RAM, each buffer is 8K
memFree = parseInt(`cat /proc/meminfo | grep MemFree | awk '{print $2}'`) # KB
if ARGV[0]
memFree = Integer(ARGV[0]) * 1000
else
memFree = parseInt(`cat /proc/meminfo | grep MemFree | awk '{print $2}'`) # KB
end
numberOfBuffers = parseInt(memFree * 0.7 / 8)
maxDirtyBuffers = numberOfBuffers / 2
puts "#{memFree} KB free, using #{numberOfBuffers} buffers, #{maxDirtyBuffers} dirty buffers"
@ -169,11 +173,22 @@ def add
}
end
def deleteAll
port = parseInt(ARGV.shift)
run "echo 'RDF_GLOBAL_RESET();' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}"
end
def stop
port = parseInt(ARGV.shift)
run "echo 'shutdown;' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}"
end
def count
port = parseInt(ARGV.shift)
run "echo 'SPARQL SELECT COUNT(*) { ?s ?p ?o };' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}"
puts "If the database is empty, this number should be 2617."
end
def status
port = parseInt(ARGV.shift)
run "echo 'status();' | #{$virtuosoPath}/install/bin/isql localhost:#{isqlPort(port)}"
@ -191,11 +206,13 @@ end
if ARGV.size == 0
puts "Usage:"
puts " #{$0} start <db directory> <port> # Starts the server on the given port"
puts " #{$0} stop <port> # Stop the server"
puts " #{$0} add <ttl file> <port> [offset] # Add new triples to the server (from offset)"
puts " #{$0} status <port> # Display server status"
puts " #{$0} shell <port> # Open shell"
puts " #{$0} start <db directory> <port> [memory (MB)] # Starts the server on the given port"
puts " #{$0} stop <port> # Stop the server"
puts " #{$0} add <ttl file> <port> [offset] # Add new triples to the server (from offset)"
puts " #{$0} deleteAll <port> # Delete the entire database (CAREFUL!)"
puts " #{$0} status <port> # Display server status"
puts " #{$0} count <port> # Display number of triples"
puts " #{$0} shell <port> # Open shell"
exit 1
end
@ -204,7 +221,9 @@ case command
when 'start' then start
when 'stop' then stop
when 'add' then add
when 'deleteAll' then deleteAll
when 'status' then status
when 'count' then count
when 'shell' then shell
else raise "Invalid command: #{command}"
end

361
module-classes.txt Normal file
View File

@ -0,0 +1,361 @@
cache edu.stanford.nlp.sempre.cache.FileStringCache
cache edu.stanford.nlp.sempre.cache.LruCallback
cache edu.stanford.nlp.sempre.cache.LruMap
cache edu.stanford.nlp.sempre.cache.RemoteStringCache
cache edu.stanford.nlp.sempre.cache.StringCache
cache edu.stanford.nlp.sempre.cache.StringCacheServer
cache edu.stanford.nlp.sempre.cache.StringCacheUtils
cache edu.stanford.nlp.sempre.cache.test.StringCacheTest
clojure edu.stanford.nlp.sempre.clojure.ClojureExecute
core edu.stanford.nlp.sempre.AbstractReinforcementParserState
core edu.stanford.nlp.sempre.AggregateFormula
core edu.stanford.nlp.sempre.ArithmeticFormula
core edu.stanford.nlp.sempre.AtomicSemType
core edu.stanford.nlp.sempre.BadFormulaException
core edu.stanford.nlp.sempre.BeamParser
core edu.stanford.nlp.sempre.BooleanValue
core edu.stanford.nlp.sempre.BoundedPriorityQueue
core edu.stanford.nlp.sempre.Builder
core edu.stanford.nlp.sempre.CallFormula
core edu.stanford.nlp.sempre.CallTypeInfo
core edu.stanford.nlp.sempre.CanonicalNames
core edu.stanford.nlp.sempre.ChartParserState
core edu.stanford.nlp.sempre.CoarseParser
core edu.stanford.nlp.sempre.Colorizer
core edu.stanford.nlp.sempre.ConcatFn
core edu.stanford.nlp.sempre.ConstantFn
core edu.stanford.nlp.sempre.ContextFn
core edu.stanford.nlp.sempre.ContextValue
core edu.stanford.nlp.sempre.Dataset
core edu.stanford.nlp.sempre.DateFn
core edu.stanford.nlp.sempre.DateValue
core edu.stanford.nlp.sempre.DerivOpCountFeatureComputer
core edu.stanford.nlp.sempre.Derivation
core edu.stanford.nlp.sempre.DerivationStream
core edu.stanford.nlp.sempre.DescriptionValue
core edu.stanford.nlp.sempre.ErrorValue
core edu.stanford.nlp.sempre.ExactValueEvaluator
core edu.stanford.nlp.sempre.Example
core edu.stanford.nlp.sempre.ExampleUtils
core edu.stanford.nlp.sempre.Executor
core edu.stanford.nlp.sempre.FeatureComputer
core edu.stanford.nlp.sempre.FeatureExtractor
core edu.stanford.nlp.sempre.FeatureMatcher
core edu.stanford.nlp.sempre.FeatureVector
core edu.stanford.nlp.sempre.FilterNerSpanFn
core edu.stanford.nlp.sempre.FilterPosTagFn
core edu.stanford.nlp.sempre.FilterSpanLengthFn
core edu.stanford.nlp.sempre.FloatingFeatureComputer
core edu.stanford.nlp.sempre.FloatingParser
core edu.stanford.nlp.sempre.FloatingRuleUtils
core edu.stanford.nlp.sempre.Formula
core edu.stanford.nlp.sempre.FormulaMatchExecutor
core edu.stanford.nlp.sempre.Formulas
core edu.stanford.nlp.sempre.FuncSemType
core edu.stanford.nlp.sempre.FuzzyMatchFn
core edu.stanford.nlp.sempre.Grammar
core edu.stanford.nlp.sempre.HasScore
core edu.stanford.nlp.sempre.IdentityFn
core edu.stanford.nlp.sempre.JavaExecutor
core edu.stanford.nlp.sempre.JoinFn
core edu.stanford.nlp.sempre.JoinFormula
core edu.stanford.nlp.sempre.Json
core edu.stanford.nlp.sempre.KnowledgeGraph
core edu.stanford.nlp.sempre.LambdaFormula
core edu.stanford.nlp.sempre.LanguageAnalyzer
core edu.stanford.nlp.sempre.LanguageInfo
core edu.stanford.nlp.sempre.Learner
core edu.stanford.nlp.sempre.ListValue
core edu.stanford.nlp.sempre.Main
core edu.stanford.nlp.sempre.MarkFormula
core edu.stanford.nlp.sempre.Master
core edu.stanford.nlp.sempre.MergeFn
core edu.stanford.nlp.sempre.MergeFormula
core edu.stanford.nlp.sempre.MultipleDerivationStream
core edu.stanford.nlp.sempre.NaiveKnowledgeGraph
core edu.stanford.nlp.sempre.NameValue
core edu.stanford.nlp.sempre.NotFormula
core edu.stanford.nlp.sempre.NullExecutor
core edu.stanford.nlp.sempre.NullTypeLookup
core edu.stanford.nlp.sempre.NumberFn
core edu.stanford.nlp.sempre.NumberValue
core edu.stanford.nlp.sempre.Params
core edu.stanford.nlp.sempre.ParaphraseModel
core edu.stanford.nlp.sempre.Parser
core edu.stanford.nlp.sempre.ParserAgenda
core edu.stanford.nlp.sempre.ParserState
core edu.stanford.nlp.sempre.PrimitiveFormula
core edu.stanford.nlp.sempre.ReinforcementParser
core edu.stanford.nlp.sempre.ReinforcementUtils
core edu.stanford.nlp.sempre.ReverseFormula
core edu.stanford.nlp.sempre.Rule
core edu.stanford.nlp.sempre.SelectFn
core edu.stanford.nlp.sempre.SemType
core edu.stanford.nlp.sempre.SemTypeHierarchy
core edu.stanford.nlp.sempre.SemanticFn
core edu.stanford.nlp.sempre.SempreUtils
core edu.stanford.nlp.sempre.Server
core edu.stanford.nlp.sempre.Session
core edu.stanford.nlp.sempre.SimpleAnalyzer
core edu.stanford.nlp.sempre.SimpleLexicon
core edu.stanford.nlp.sempre.SimpleLexiconFn
core edu.stanford.nlp.sempre.SingleDerivationStream
core edu.stanford.nlp.sempre.StringValue
core edu.stanford.nlp.sempre.SuperlativeFormula
core edu.stanford.nlp.sempre.TableValue
core edu.stanford.nlp.sempre.TopSemType
core edu.stanford.nlp.sempre.Trie
core edu.stanford.nlp.sempre.TypeInference
core edu.stanford.nlp.sempre.TypeLookup
core edu.stanford.nlp.sempre.UnionSemType
core edu.stanford.nlp.sempre.UriValue
core edu.stanford.nlp.sempre.Value
core edu.stanford.nlp.sempre.ValueEvaluator
core edu.stanford.nlp.sempre.ValueFormula
core edu.stanford.nlp.sempre.Values
core edu.stanford.nlp.sempre.VariableFormula
core edu.stanford.nlp.sempre.test.DerivationStreamTest
core edu.stanford.nlp.sempre.test.FormulaTest
core edu.stanford.nlp.sempre.test.GrammarTest
core edu.stanford.nlp.sempre.test.GrammarValidityTest
core edu.stanford.nlp.sempre.test.JavaExecutorTest
core edu.stanford.nlp.sempre.test.JsonTest
core edu.stanford.nlp.sempre.test.ParserTest
core edu.stanford.nlp.sempre.test.SemTypeTest
core edu.stanford.nlp.sempre.test.SemanticFnTest
core edu.stanford.nlp.sempre.test.SystemSanityTest
core edu.stanford.nlp.sempre.test.TestUtils
core edu.stanford.nlp.sempre.test.TypeInferenceTest
corenlp edu.stanford.nlp.sempre.corenlp.CoreNLPAnalyzer
corenlp edu.stanford.nlp.sempre.corenlp.test.CoreNLPSemanticFnTest
fbalignment edu.stanford.nlp.sempre.fbalignment.BinaryLexiconConstructionMain
fbalignment edu.stanford.nlp.sempre.fbalignment.FreebaseAlignmentDataManager
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.BipartiteBuilder
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.FromFbGraphBuilder
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.classify.ClassifierUtils
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.classify.DatumGenerator
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.BipartiteNodeFilter
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.FbRelationFilter
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.NodeFrequencyFilter
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.filters.NodeFrequencyOrFbRelationFilter
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.AlignmentLearner
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.AlignmentLearnerFactory
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ChooseBestRelation
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ChooseBestTypeCompatibleRelation
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ExhaustiveNlTyping
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.ExhaustiveNlTypingNoCommit
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.FbBasedNlTyping
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.NlIterLearner
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.learner.SplitPredicateToDisjointTypes
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteEdge
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteEdgeFactory
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteGraph
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteNode
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteNodeFactory
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.BipartiteNodeType
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.CompositeFbFact
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.CountEdge
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.FbBipartiteNode
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.FbTypedBipartiteNode
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.FourPartiteGraph
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.MatchSampleAndCountEdge
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.NlBipartiteNode
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.NlTypedBipartiteNode
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.ScoreEdge
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.rep.UnlabeledEdge
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.CountScorer
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.MaxentScorer
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.NodePairScorer
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.ScorerFactory
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.SmoothedJaccardAndCountThresholdScorer
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.scorers.SmoothedJaccardScorer
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.AlignmentExample
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.AlignmentExperimentManager
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.AnnotatedAlignmentExample
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.HasLabel
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.test.PRCurve
fbalignment edu.stanford.nlp.sempre.fbalignment.bipartite.visualization.Visualizer
fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FbEntity
fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FbGraphBuilder
fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FbPropertiesExpectedTypes
fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FromPartiallyLinkedExtractionsGraphBuilder
fbalignment edu.stanford.nlp.sempre.fbalignment.fbgraph.FromPartiallyLinkedThroughCvtFbGraphBuilder
fbalignment edu.stanford.nlp.sempre.fbalignment.jung.DirectedSparseGraphVertexAccess
fbalignment edu.stanford.nlp.sempre.fbalignment.jung.UndirectedSparseGraphVertexAccess
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.ArgSubstringMatcher
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.ExactMatcher
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.IgnoreCaseAndTimeMatcher
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.IgnoreCaseMatcher
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.Matcher
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.MatcherFactory
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.SymmetricSubstringMatcher
fbalignment edu.stanford.nlp.sempre.fbalignment.matchers.TimeMatcher
fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.BinaryExtractionsPreprocessor
fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.ExtractionsPreprocessor
fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.LinkedExtractionsPreprocessMain
fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.LinkedTimeExtractionCreator
fbalignment edu.stanford.nlp.sempre.fbalignment.preprocess_openie.UnaryExtractionsPreprocessor
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.AddAlignmentScoresToPropertyInfoFile
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.AlignmentBinaryLexicon
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.EntityTypesAdder
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.ExtractResultsFromLogFile
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.FilterEntityFileByName
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateAtomicBinaryFormulaInfo
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateContainedByNameds
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateCvts
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateDataFromLog
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateEntityInfoFile
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateExpectedTypes
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GeneratePropertyFile
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateTypeToId
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.GenerateUnaryInfoFile
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.MidToIdFixer
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.MidToIdReplacer
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.SlashToDotConverter
fbalignment edu.stanford.nlp.sempre.fbalignment.scripts.SplitExamples
fbalignment edu.stanford.nlp.sempre.fbalignment.testers.BipartiteGraphStatsTester
fbalignment edu.stanford.nlp.sempre.fbalignment.testers.FourPartiteLearnerTester
fbalignment edu.stanford.nlp.sempre.fbalignment.testers.LexiconTester
fbalignment edu.stanford.nlp.sempre.fbalignment.testers.LinearClassifierFromAnnotationFile
fbalignment edu.stanford.nlp.sempre.fbalignment.testers.UnaryInfoTester
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.FbUnaryAlignmentNode
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.FbUnaryNodeFilter
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.NlUnaryAlignmentNode
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryAlignmentManager
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryAlignmentNode
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryInfoData
fbalignment edu.stanford.nlp.sempre.fbalignment.unary.UnaryTupleGenerator
freebase edu.stanford.nlp.sempre.freebase.BinaryLexicon
freebase edu.stanford.nlp.sempre.freebase.BridgeFn
freebase edu.stanford.nlp.sempre.freebase.BuildCanonicalIdMap
freebase edu.stanford.nlp.sempre.freebase.BuildTypesMap
freebase edu.stanford.nlp.sempre.freebase.CanonicalizeExamples
freebase edu.stanford.nlp.sempre.freebase.CanonicalizeIds
freebase edu.stanford.nlp.sempre.freebase.EntityLexicon
freebase edu.stanford.nlp.sempre.freebase.ExecuteExamples
freebase edu.stanford.nlp.sempre.freebase.FbFormulasInfo
freebase edu.stanford.nlp.sempre.freebase.FilterFreebase
freebase edu.stanford.nlp.sempre.freebase.Free917Converter
freebase edu.stanford.nlp.sempre.freebase.FreebaseInfo
freebase edu.stanford.nlp.sempre.freebase.FreebaseSearch
freebase edu.stanford.nlp.sempre.freebase.FreebaseTypeLookup
freebase edu.stanford.nlp.sempre.freebase.FreebaseValueEvaluator
freebase edu.stanford.nlp.sempre.freebase.LambdaCalculusConverter
freebase edu.stanford.nlp.sempre.freebase.Lexicon
freebase edu.stanford.nlp.sempre.freebase.LexiconFn
freebase edu.stanford.nlp.sempre.freebase.SparqlExecutor
freebase edu.stanford.nlp.sempre.freebase.SparqlExpr
freebase edu.stanford.nlp.sempre.freebase.Stemmer
freebase edu.stanford.nlp.sempre.freebase.TextToTextMatcher
freebase edu.stanford.nlp.sempre.freebase.UnaryLexicon
freebase edu.stanford.nlp.sempre.freebase.Utils
freebase edu.stanford.nlp.sempre.freebase.index.FbEntityIndexer
freebase edu.stanford.nlp.sempre.freebase.index.FbEntitySearcher
freebase edu.stanford.nlp.sempre.freebase.index.FbIndexField
freebase edu.stanford.nlp.sempre.freebase.lexicons.EntrySource
freebase edu.stanford.nlp.sempre.freebase.lexicons.ExtremeValueWrapper
freebase edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry
freebase edu.stanford.nlp.sempre.freebase.lexicons.TokenLevelMatchFeatures
freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.BinaryNormalizer
freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.EntryNormalizer
freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.IdentityNormalizer
freebase edu.stanford.nlp.sempre.freebase.lexicons.normalizers.PrepDropNormalizer
freebase edu.stanford.nlp.sempre.freebase.test.FbFormulasTest
freebase edu.stanford.nlp.sempre.freebase.test.FreebaseInfoTest
freebase edu.stanford.nlp.sempre.freebase.test.FreebaseSemTypeTest
freebase edu.stanford.nlp.sempre.freebase.test.FreebaseTypeInferenceTest
freebase edu.stanford.nlp.sempre.freebase.test.LambdaCalculusConverterTest
freebase edu.stanford.nlp.sempre.freebase.test.LexiconTest
freebase edu.stanford.nlp.sempre.freebase.test.PrepDropNormalizerTest
freebase edu.stanford.nlp.sempre.freebase.test.SparqlExecutorTest
freebase edu.stanford.nlp.sempre.freebase.test.StemmerTest
freebase edu.stanford.nlp.sempre.freebase.test.TokenMatchTest
freebase edu.stanford.nlp.sempre.freebase.utils.CollectionUtils
freebase edu.stanford.nlp.sempre.freebase.utils.DoubleContainer
freebase edu.stanford.nlp.sempre.freebase.utils.FileUtils
freebase edu.stanford.nlp.sempre.freebase.utils.FormatConverter
freebase edu.stanford.nlp.sempre.freebase.utils.FreebaseUtils
freebase edu.stanford.nlp.sempre.freebase.utils.LinkedExtractionFileUtils
freebase edu.stanford.nlp.sempre.freebase.utils.MathUtils
freebase edu.stanford.nlp.sempre.freebase.utils.SemparseLogTools
freebase edu.stanford.nlp.sempre.freebase.utils.ShortContainer
freebase edu.stanford.nlp.sempre.freebase.utils.WnExpander
freebase edu.stanford.nlp.sempre.freebase.utils.WordNet
jungle edu.stanford.nlp.sempre.jungle.WebformExecutor
paraphrase edu.stanford.nlp.sempre.paraphrase.Aligner
paraphrase edu.stanford.nlp.sempre.paraphrase.Context
paraphrase edu.stanford.nlp.sempre.paraphrase.DatasetUtils
paraphrase edu.stanford.nlp.sempre.paraphrase.FeatureSimilarity
paraphrase edu.stanford.nlp.sempre.paraphrase.FeatureSimilarityComputer
paraphrase edu.stanford.nlp.sempre.paraphrase.FormulaGenerationInfo
paraphrase edu.stanford.nlp.sempre.paraphrase.FormulaRetriever
paraphrase edu.stanford.nlp.sempre.paraphrase.Interval
paraphrase edu.stanford.nlp.sempre.paraphrase.NnBuilder
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseBuilder
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseDataset
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseDerivation
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseExample
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseFeatureExtractor
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseFeatureMatcher
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseLearner
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseMain
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseParser
paraphrase edu.stanford.nlp.sempre.paraphrase.ParaphraseUtils
paraphrase edu.stanford.nlp.sempre.paraphrase.ParsingExample
paraphrase edu.stanford.nlp.sempre.paraphrase.ParsingExampleProcessor
paraphrase edu.stanford.nlp.sempre.paraphrase.Prediction
paraphrase edu.stanford.nlp.sempre.paraphrase.Proof
paraphrase edu.stanford.nlp.sempre.paraphrase.ProofScorer
paraphrase edu.stanford.nlp.sempre.paraphrase.ProofTransformationModel
paraphrase edu.stanford.nlp.sempre.paraphrase.QuestionGenerator
paraphrase edu.stanford.nlp.sempre.paraphrase.Transformer
paraphrase edu.stanford.nlp.sempre.paraphrase.VectorSpaceModel
paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParalexEditDistanceComputer
paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParalexQuestionReader
paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParalexRules
paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.ParaphraseTableExtractor
paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.PhraseTable
paraphrase edu.stanford.nlp.sempre.paraphrase.paralex.PosNerTagger
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LangItem
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LanguageExp
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LanguageExpToken
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosRule
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.LemmaPosSequence
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.ParaphraseAlignment
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.RuleApplication
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.RuleApplier
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.Rulebase
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.SubstitutionRuleExtractor
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.SyntacticRuleSet
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.VerbSemClassExtractor
paraphrase edu.stanford.nlp.sempre.paraphrase.rules.VerbSemClassMatcher
paraphrase edu.stanford.nlp.sempre.paraphrase.scripts.CompareLogFiles
paraphrase edu.stanford.nlp.sempre.paraphrase.scripts.ExtractTrainTrueAndPredLines
paraphrase edu.stanford.nlp.sempre.paraphrase.scripts.GenerateOracleRelationsAndEntities
regex edu.stanford.nlp.sempre.regex.RegexValueEvaluator
tables edu.stanford.nlp.sempre.tables.ConvertCsvToTtl
tables edu.stanford.nlp.sempre.tables.DPParser
tables edu.stanford.nlp.sempre.tables.DPParserPruner
tables edu.stanford.nlp.sempre.tables.FuzzyMatcher
tables edu.stanford.nlp.sempre.tables.StringNormalizationUtils
tables edu.stanford.nlp.sempre.tables.TableKnowledgeGraph
tables edu.stanford.nlp.sempre.tables.TableTypeLookup
tables edu.stanford.nlp.sempre.tables.TableTypeSystem
tables edu.stanford.nlp.sempre.tables.TableValueEvaluator
tables edu.stanford.nlp.sempre.tables.lambdadcs.BinaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.BinaryTypeHint
tables edu.stanford.nlp.sempre.tables.lambdadcs.DenotationUtils
tables edu.stanford.nlp.sempre.tables.lambdadcs.ExplicitBinaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.ExplicitUnaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.InfiniteUnaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSException
tables edu.stanford.nlp.sempre.tables.lambdadcs.LambdaDCSExecutor
tables edu.stanford.nlp.sempre.tables.lambdadcs.PredicateBinaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.SpecialBinaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.TypeHint
tables edu.stanford.nlp.sempre.tables.lambdadcs.UnaryDenotation
tables edu.stanford.nlp.sempre.tables.lambdadcs.UnaryTypeHint
tables edu.stanford.nlp.sempre.tables.test.AnnotatedFormulasChecker
tables edu.stanford.nlp.sempre.tables.test.CustomExample
tables edu.stanford.nlp.sempre.tables.test.DPParserChecker
tables edu.stanford.nlp.sempre.tables.test.LambdaDCSExecutorTest

178
parasempre
View File

@ -1,178 +0,0 @@
#!/usr/bin/ruby
$: << 'fig/lib'
require 'execrunner'
# Note: run this on the NLP machines because SPARQL server is running on jack.
if ARGV.size == 0
puts "Usage:"
puts " ./parasempre @mode=train @domain=<webquestions|free917> @sparqlserver=<host:port> @cacheserver=<none|local>"
puts "Additional options can be any of the following:"
puts " - Additional program options (e.g., -BeamParser.beamSize 3)"
puts " - Execrunner options, which select the options to include (@data=0)"
exit 1
end
system "mkdir -p trans_state/execs"
system "touch trans_state/lastExec"
def header
l(
letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
'fig/bin/qcreate',
'-statePath',
'trans_state',
(File.exists?('/u/nlp/bin/java7') ? '/u/nlp/bin/java7' : 'java'),
'-ea',
'-Xmx10g',
'-cp', 'classes:'+Dir['lib/*.jar'].join(':'),
letDefault(:prof, 0), sel(:prof,
l(),
'-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt',
'-Xrunhprof:heap=sites,file=_OUTPATH_/java.hprof.txt'),
nil)
end
def sparqlOpts
l(
required(:sparqlserver, 'host:port of the Sparql server'),
o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}),
nil)
end
def defaultOpts
l(
o('execDir', '_OUTPATH_'), o('overwriteExecDir'),
o('addToView', 0),
nil)
end
def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath)
l(
required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or <hostname>:<port> (hit the cacheserver)'),
lambda { |e|
cacheserver = e[:cacheserver]
case cacheserver
when 'none' then l()
when 'local' then l( # Use files directly - don't run more than one job that does this!
o('Lexicon.cachePath', 'LexiconFn.cache'),
o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'),
nil)
else l(
o('Lexicon.cachePath', cacheserver + ':' + lexiconFnCachePath),
o('SparqlExecutor.cachePath', cacheserver + ':' + sparqlExecutorCachePath),
nil)
end
},
nil)
end
$defaultFeatureDomains = [
'Del',
'Denotation',
'Formula',
'NamedEntity',
#'Pt',
'Subst',
'WhType',
nil].compact
def unbalancedTrainDevSplit
l(o('ParaphraseDataset.trainFrac', 0.8), o('ParaphraseDataset.devFrac', 0.2))
end
def allTrainSplit
l(o('ParaphraseDataset.trainFrac', 1), o('ParaphraseDataset.devFrac', 0))
end
def train
l(
header,
'edu.stanford.nlp.sempre.paraphrase.ParaphraseMain',
defaultOpts, sparqlOpts,
o('ParaphraseMain.mode','train'),
required(:domain, 'domain (webquestions or free917)'),
selectDomain,
o('Rulebase.ruleTypes','Pt','Syntax', 'Move', 'Subst'),
o('ParaphraseFeatureMatcher.featureDomains', *$defaultFeatureDomains),
o('Params.l1Reg','lazy'),
o('Aligner.useWordnet',true),
o('FeatureSimilarityComputer.mode','lexical_overlap'),
nil)
end
def webquestions
l(
letDefault(:data, 0),
sel(:data,
l(o('ParaphraseDataset.parsingInPaths',
'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'),
unbalancedTrainDevSplit,
nil),
l(o('ParaphraseDataset.parsingInPaths',
'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json',
'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'),
allTrainSplit,
nil),
nil),
o('ParaphraseLearner.maxTrainIters',2),
o('ParaphraseLearner.partialReward',true),
o('ParaphraseParser.alignment',true),
o('ParaphraseParser.vsm',true),
o('Lexicon.entitySearchStrategy','inexact'),
o('FormulaRetriever.supportCountUtterances',false),
o('FormulaRetriever.filterRelations',true),
o('FormulaRetriever.maxEntries',10),
o('FormulaRetriever.conservativeEntityExtraction',true),
o('Params.l1RegCoeff','0.00017782794'),
o('VectorSpaceModel.wordVectorFile','lib/wordreprs/cbow-lowercase-50.vectors'),
l(
o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/inexact/'),
cachePaths('LexiconFnWebQ.cache', 'SparqlExecutor.cache'),
nil),
nil)
end
def free917
l(
letDefault(:data, 0),
sel(:data,
l(o('ParaphraseDataset.parsingInPaths', 'train,data/free917.train.examples.canonicalized.json'),
unbalancedTrainDevSplit,
nil),
l(o('ParaphraseDataset.parsingInPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json'),
allTrainSplit,
nil),
nil),
o('ParaphraseLearner.maxTrainIters',10),
o('ParaphraseLearner.partialReward',false),
o('ParaphraseParser.alignment',true),
o('ParaphraseParser.vsm',true),
o('Lexicon.entitySearchStrategy','exact'),
o('FormulaRetriever.supportCountUtterances',true),
o('FormulaRetriever.filterRelations',false),
o('FormulaRetriever.maxEntries',1000),
o('FormulaRetriever.conservativeEntityExtraction',false),
o('Params.l1RegCoeff',0.00562341325),
o('VectorSpaceModel.wordVectorFile','lib/wordreprs/wordLexicon'),
l(
o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'),
cachePaths('LexiconFnFree917.cache', 'SparqlExecutor.cache'),
nil),
nil)
end
def selectDomain
sel(:domain, {
'webquestions' => webquestions,
'free917' => free917,
})
end
run!(
sel(:mode, {
'train' => train,
}),
nil)

163
pull-dependencies Executable file
View File

@ -0,0 +1,163 @@
#!/usr/bin/env ruby
# SEMPRE depends on several library/data files into *lib*. Run this script to
# copy those dependencies to your local directory. This allows you to run
# SEMPRE from anywhere. This file consists of a set of modules (which loosely
# correspond to the code modules).
# The master copy of these dependencies are stored on the Stanford NLP
# machines.
$version = '2.0'
$isPublic = true
def isZip(name)
# Directories are zipped
name.end_with?('.exec') or name !~ /\./
end
# - Download the dependencies from the Stanford SEMPRE servers.
def pull(sourcePath, dir=nil, opts={})
puts sourcePath
destDir = 'lib' + (dir ? '/' + dir : '')
system "mkdir -p #{destDir}"
name = File.basename(sourcePath)
ext = isZip(name) ? '.zip' : ''
if $isPublic
# Download url => localPath
url = 'http://nlp.stanford.edu/software/sempre/dependencies-' + $version + sourcePath + ext
localPath = destDir + '/' + name + ext
system "mkdir -p #{File.dirname(localPath)}" or exit 1
system "wget -c '#{url}' -O #{localPath}" or exit 1
# Unzip localPath to destDir if it's a zip file
if isZip(name)
system "cd #{File.dirname(localPath)} && unzip #{File.basename(localPath)}" or exit 1
system "rm #{localPath}" or exit 1
end
else
end
end
# source: path to master git repository
def updateGit(source)
dir = File.basename(source.sub(/\.git$/, ''))
if File.exists?(dir)
system 'cd '+dir+' && git pull' or exit 1
else
system 'git clone ' + source or exit 1
end
end
def downloadExec(path)
['options.map', 'params', 'grammar'].each { |file|
if File.exists?(path+'/'+file)
pull(path+'/'+file, 'models/'+File.basename(path))
end
}
end
$modules = []
def addModule(name, description, func)
$modules << [name, description, func]
end
############################################################
addModule('core', 'Core utilities (need to compile)', lambda {
# fig: options parsing, experiment management, utils
updateGit('https://github.com/percyliang/fig')
system 'make -C fig' or exit 1
system 'mkdir -p lib && cd lib && ln -sf ../fig/fig.jar' or exit 1
# Google libraries
pull('/u/nlp/data/semparse/resources/guava-14.0.1.jar')
# TestNG -- testing framework
pull('/u/nlp/data/semparse/resources/testng-6.8.5.jar')
pull('/u/nlp/data/semparse/resources/jcommander-1.30.jar')
# Checkstyle: make sure code looks fine
pull('/u/nlp/data/semparse/resources/checkstyle')
# JSON
pull('/u/nlp/data/semparse/resources/jackson-core-2.2.0.jar')
pull('/u/nlp/data/semparse/resources/jackson-annotations-2.2.0.jar')
pull('/u/nlp/data/semparse/resources/jackson-databind-2.2.0.jar')
})
addModule('corenlp', 'Stanford CoreNLP (code and modules)', lambda {
pull('/u/nlp/data/semparse/resources/stanford-corenlp-full-2013-06-20.zip', '', {:symlink => true})
if not File.exists?('lib/stanford-corenlp-full-2013-06-20')
system "cd lib && unzip stanford-corenlp-full-2013-06-20.zip" or exit 1
end
['stanford-corenlp-3.2.0.jar', 'stanford-corenlp-3.2.0-models.jar', 'joda-time.jar', 'jollyday.jar'].each { |file|
system "ln -sfv stanford-corenlp-full-2013-06-20/#{file} lib" or exit 1
}
pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2013-06-07-models.jar', '', {:symlink => true})
})
addModule('freebase', 'Freebase: need to construct Freebase schemas', lambda {
# Freebase schema
pull('/u/nlp/data/semparse/scr/freebase/state/execs/93.exec/schema2.ttl', 'fb_data/93.exec')
# Lucene libraries
pull('/u/nlp/data/semparse/resources/lucene-core-4.4.0.jar')
pull('/u/nlp/data/semparse/resources/lucene-analyzers-common-4.4.0.jar')
pull('/u/nlp/data/semparse/resources/lucene-queryparser-4.4.0.jar')
# Freebase data (for lexicon)
pull('/u/nlp/data/semparse/scr/fb_data/7', 'fb_data')
# WebQuestions dataset
pull('/u/nlp/data/semparse/webquestions/dataset_11/webquestions.examples.train.json', 'data/webquestions/dataset_11')
pull('/u/nlp/data/semparse/webquestions/dataset_11/webquestions.examples.test.json', 'data/webquestions/dataset_11')
})
addModule('virtuoso', 'Virtuoso: if want to run own SPARQL server locally', lambda {
updateGit('https://github.com/openlink/virtuoso-opensource')
# Run this command to compile:
#system "cd virtuoso-opensource && ./autogen.sh && ./configure --prefix=$PWD/install && make && make install" or exit 1
})
addModule('fullfreebase-ttl', 'Freebase (ttl file)', lambda {
# This is just for your reference. It is not directly used by SEMPRE.
pull('/u/nlp/data/semparse/scr/freebase/state/execs/93.exec/0.ttl.bz2', 'fb_data/93.exec', {:symlink => true})
})
addModule('fullfreebase-vdb', 'Freebase (Virtuoso database)', lambda {
# Virtuoso index of 0.ttl above. This is read (and written) by Virtuoso.
pull('/u/nlp/data/semparse/scr/freebase/state/execs/93.exec/vdb.tar.bz2', 'fb_data/93.exec', {:symlink => true})
# You need to unzip this yourself and move these files to the right place.
})
addModule('fullfreebase-types', 'Freebase types', lambda {
# Map from mid (e.g., fb:m.02mjmr) to id (e.g., fb:en.barack_obama) (used to link external things like Freebase API search with our internal Freebase)
pull('/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map.gz', 'fb_data', {:symlink => true})
# Map from id to types (used to do type inference on internal entities)
pull('/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized.en-types.gz', 'fb_data', {:symlink => true})
# You need to unzip these yourself and move these files to the right place.
})
############################################################
if ARGV.size == 0
puts "#{$0} <module-1> ... <module-n>"
puts
puts "Modules:"
$modules.each { |name,description,func|
puts " #{name}: #{description}"
}
end
$modules.each { |name,description,func|
if ARGV.index(name)
puts "===== Downloading #{name}: #{description}"
func.call
end
}

46
release-acl2014.files
View File

@ -1,46 +0,0 @@
# This file contains additional public datasets described in the EMNLP
# 2013 paper, including WebQuestions.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies emnlp2013
#
############################################################
#wordnet
lib/wordnet-3.0-prolog
lib/derivation.txt
#word vectors
lib/wordreprs/cbow-lowercase-50.vectors
lib/wordreprs/wordLexicon
#paralex phrase table
lib/paralex/phrase-table.counts.txt
# Lexicon alignment data
lib/fb_data/6/unaryInfoStringAndAlignment.txt
lib/fb_data/6/binaryInfoStringAndAlignment.txt
############################################################
# WebQuestions
lib/data/webquestions/dataset_11/webquestions.examples.train.json
lib/data/webquestions/dataset_11/webquestions.examples.test.json
############################################################
# Free917 [Cai & Yates, 2013]
data/free917.train.examples.canonicalized.json
data/free917.test.examples.canonicalized.json
############################################################
# Lucene
lib/lucene/4.4/free917
lib/lucene/4.4/inexact
############################################################
# Model
lib/models/915.exec # test model for acl 2014

244
release-code.files
View File

@ -1,244 +0,0 @@
# This file contains all the source files included in the public
# code release.
#
# These files go into the public github repo.
############################################################
release-code.files
release-core.files
release-emnlp2013.files
release-emnlp2014.files
release-fullfreebase_ttl.files
release-fullfreebase_vdb.files
release-geofreebase_ttl.files
release-geofreebase_vdb.files
sempre
parasempre
Makefile
README.md
LICENSE.txt
TUTORIAL.md
QUICKSTART.md
testng.xml
download-dependencies
# Ruby utils: execrunner script etc.
fig/bin/qcreate
fig/bin/chunk
fig/lib/execrunner.rb
fig/lib/myutils.rb
# Main
src/edu/stanford/nlp/sempre/Main.java
src/edu/stanford/nlp/sempre/Master.java
src/edu/stanford/nlp/sempre/Session.java
src/edu/stanford/nlp/sempre/Builder.java
src/edu/stanford/nlp/sempre/Dataset.java
src/edu/stanford/nlp/sempre/Learner.java
src/edu/stanford/nlp/sempre/Params.java
src/edu/stanford/nlp/sempre/FeatureVector.java
src/edu/stanford/nlp/sempre/FeatureMatcher.java
src/edu/stanford/nlp/sempre/Evaluation.java
# Parsing
src/edu/stanford/nlp/sempre/Parsers.java
src/edu/stanford/nlp/sempre/Parser.java
src/edu/stanford/nlp/sempre/ParserState.java
src/edu/stanford/nlp/sempre/BeamParser.java
src/edu/stanford/nlp/sempre/Grammar.java
src/edu/stanford/nlp/sempre/Rule.java
src/edu/stanford/nlp/sempre/Derivation.java
src/edu/stanford/nlp/sempre/FeatureExtractor.java
src/edu/stanford/nlp/sempre/DerivationConstraint.java
src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java
src/edu/stanford/nlp/sempre/Example.java
src/edu/stanford/nlp/sempre/TextToTextMatcher.java
src/edu/stanford/nlp/sempre/FormulaRetriever.java
src/edu/stanford/nlp/sempre/FormulaGenerationInfo.java
# Support
src/edu/stanford/nlp/sempre/LanguageInfo.java
src/edu/stanford/nlp/sempre/FbFormulasInfo.java
src/edu/stanford/nlp/sempre/FreebaseInfo.java
src/edu/stanford/nlp/sempre/StringCacheServer.java
src/edu/stanford/nlp/sempre/StringCacheUtils.java
src/edu/stanford/nlp/sempre/Json.java
src/edu/stanford/nlp/sempre/Stemmer.java
src/edu/stanford/nlp/sempre/StringCache.java
src/edu/stanford/nlp/sempre/Trie.java
src/edu/stanford/nlp/sempre/FreebaseSearch.java
# Formula
src/edu/stanford/nlp/sempre/Formulas.java
src/edu/stanford/nlp/sempre/Formula.java
src/edu/stanford/nlp/sempre/PrimitiveFormula.java
src/edu/stanford/nlp/sempre/ValueFormula.java
src/edu/stanford/nlp/sempre/JoinFormula.java
src/edu/stanford/nlp/sempre/MergeFormula.java
src/edu/stanford/nlp/sempre/VariableFormula.java
src/edu/stanford/nlp/sempre/LambdaFormula.java
src/edu/stanford/nlp/sempre/MarkFormula.java
src/edu/stanford/nlp/sempre/ReverseFormula.java
src/edu/stanford/nlp/sempre/AggregateFormula.java
src/edu/stanford/nlp/sempre/SuperlativeFormula.java
src/edu/stanford/nlp/sempre/NotFormula.java
src/edu/stanford/nlp/sempre/CallFormula.java
src/edu/stanford/nlp/sempre/SemType.java
# Value
src/edu/stanford/nlp/sempre/Values.java
src/edu/stanford/nlp/sempre/Value.java
src/edu/stanford/nlp/sempre/NumberValue.java
src/edu/stanford/nlp/sempre/StringValue.java
src/edu/stanford/nlp/sempre/DateValue.java
src/edu/stanford/nlp/sempre/UriValue.java
src/edu/stanford/nlp/sempre/ErrorValue.java
src/edu/stanford/nlp/sempre/ListValue.java
src/edu/stanford/nlp/sempre/NameValue.java
src/edu/stanford/nlp/sempre/BooleanValue.java
src/edu/stanford/nlp/sempre/DescriptionValue.java
# SemanticFn
src/edu/stanford/nlp/sempre/SemanticFn.java
src/edu/stanford/nlp/sempre/IdentityFn.java
src/edu/stanford/nlp/sempre/ConstantFn.java
src/edu/stanford/nlp/sempre/NumberFn.java
src/edu/stanford/nlp/sempre/ConcatFn.java
src/edu/stanford/nlp/sempre/DateFn.java
src/edu/stanford/nlp/sempre/LexiconFn.java
src/edu/stanford/nlp/sempre/JoinFn.java
src/edu/stanford/nlp/sempre/MergeFn.java
src/edu/stanford/nlp/sempre/SelectFn.java
src/edu/stanford/nlp/sempre/FilterPosTagFn.java
src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java
src/edu/stanford/nlp/sempre/FilterNerSpanFn.java
src/edu/stanford/nlp/sempre/BridgeFn.java
# Executor
src/edu/stanford/nlp/sempre/Executor.java
src/edu/stanford/nlp/sempre/NullExecutor.java
src/edu/stanford/nlp/sempre/JavaExecutor.java
src/edu/stanford/nlp/sempre/FormulaMatchExecutor.java
src/edu/stanford/nlp/sempre/SparqlExecutor.java
src/edu/stanford/nlp/sempre/SparqlExpr.java
# Lexicon
src/edu/stanford/nlp/sempre/fbalignment/lexicons/Lexicon.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntityLexicon.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/BinaryLexicon.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/UnaryLexicon.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/LexicalEntry.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/EntrySource.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/ExtremeValueWrapper.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/TokenLevelMatchFeatures.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/WordDistance.java
# Lexicon normalizers
src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/EntryNormalizer.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/IdentityNormalizer.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/PrepDropNormalizer.java
src/edu/stanford/nlp/sempre/fbalignment/lexicons/normalizers/BinaryNormalizer.java
# Lucene indexing
src/edu/stanford/nlp/sempre/fbalignment/index/FbEntitySearcher.java
src/edu/stanford/nlp/sempre/fbalignment/index/FbEntityIndexer.java
src/edu/stanford/nlp/sempre/fbalignment/index/FbIndexField.java
# FbAlignment
src/edu/stanford/nlp/sempre/fbalignment/utils/DoubleContainer.java
src/edu/stanford/nlp/sempre/fbalignment/utils/FileUtils.java
src/edu/stanford/nlp/sempre/fbalignment/utils/MathUtils.java
src/edu/stanford/nlp/sempre/fbalignment/utils/CollectionUtils.java
src/edu/stanford/nlp/sempre/fbalignment/utils/WnExpander.java
src/edu/stanford/nlp/sempre/fbalignment/utils/WordNet.java
# Freebase
src/edu/stanford/nlp/sempre/freebase/Utils.java
# Tests
src/edu/stanford/nlp/sempre/test/FbFormulasTest.java
src/edu/stanford/nlp/sempre/test/FormulaTest.java
src/edu/stanford/nlp/sempre/test/JavaExecutorTest.java
src/edu/stanford/nlp/sempre/test/JsonTest.java
src/edu/stanford/nlp/sempre/test/LexiconTest.java
src/edu/stanford/nlp/sempre/test/ParserTest.java
src/edu/stanford/nlp/sempre/test/PrepDropNormalizerTest.java
src/edu/stanford/nlp/sempre/test/SemTypeTest.java
src/edu/stanford/nlp/sempre/test/SemanticFnTest.java
src/edu/stanford/nlp/sempre/test/SparqlExecutorTest.java
src/edu/stanford/nlp/sempre/test/StemmerTest.java
src/edu/stanford/nlp/sempre/test/SystemSanityTest.java
src/edu/stanford/nlp/sempre/test/TestUtils.java
src/edu/stanford/nlp/sempre/test/TokenMatchTest.java
# Vis
src/edu/stanford/nlp/sempre/Vis.java
src/edu/stanford/nlp/sempre/vis/BeamFigures.java
src/edu/stanford/nlp/sempre/vis/ConfusionMatrices.java
src/edu/stanford/nlp/sempre/vis/ExampleDerivations.java
src/edu/stanford/nlp/sempre/vis/Utils.java
# Paraphrasing
src/edu/stanford/nlp/sempre/paraphrase/Aligner.java
src/edu/stanford/nlp/sempre/paraphrase/Context.java
src/edu/stanford/nlp/sempre/paraphrase/Context.java
src/edu/stanford/nlp/sempre/paraphrase/ContextModel.java
src/edu/stanford/nlp/sempre/paraphrase/ContextSimilarityModel.java
src/edu/stanford/nlp/sempre/paraphrase/DatasetUtils.java
src/edu/stanford/nlp/sempre/paraphrase/EntityInstance.java
src/edu/stanford/nlp/sempre/paraphrase/EntityModel.java
src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarity.java
src/edu/stanford/nlp/sempre/paraphrase/FeatureSimilarityComputer.java
src/edu/stanford/nlp/sempre/paraphrase/Interval.java
src/edu/stanford/nlp/sempre/paraphrase/NearestNeighborLearner.java
src/edu/stanford/nlp/sempre/paraphrase/NnBuilder.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseBuilder.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDataset.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseDerivation.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseExample.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureExtractor.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseFeatureMatcher.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseLearner.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseMain.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseParser.java
src/edu/stanford/nlp/sempre/paraphrase/ParaphraseUtils.java
src/edu/stanford/nlp/sempre/paraphrase/ParsingExample.java
src/edu/stanford/nlp/sempre/paraphrase/ParsingExampleProcessor.java
src/edu/stanford/nlp/sempre/paraphrase/Prediction.java
src/edu/stanford/nlp/sempre/paraphrase/QuestionGenerator.java
src/edu/stanford/nlp/sempre/paraphrase/VectorSpaceModel.java
src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexQuestionReader.java
src/edu/stanford/nlp/sempre/paraphrase/paralex/ParalexRules.java
src/edu/stanford/nlp/sempre/paraphrase/paralex/PhraseTable.java
src/edu/stanford/nlp/sempre/paraphrase/rules/LangItem.java
src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExp.java
src/edu/stanford/nlp/sempre/paraphrase/rules/LanguageExpToken.java
src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosRule.java
src/edu/stanford/nlp/sempre/paraphrase/rules/LemmaPosSequence.java
src/edu/stanford/nlp/sempre/paraphrase/rules/ParaphraseAlignment.java
src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplication.java
src/edu/stanford/nlp/sempre/paraphrase/rules/RuleApplier.java
src/edu/stanford/nlp/sempre/paraphrase/rules/Rulebase.java
src/edu/stanford/nlp/sempre/paraphrase/rules/SubstitutionRuleExtractor.java
src/edu/stanford/nlp/sempre/paraphrase/rules/SyntacticRuleSet.java
src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassExtractor.java
src/edu/stanford/nlp/sempre/paraphrase/rules/VerbSemClassMatcher.java
# Create Freebase
scripts/extract-freebase-schema.rb
scripts/virtuoso
############################################################
# Really simple data
data/unittest-learn.grammar
data/unittest-learn-ccg.grammar
data/unittest-learn.examples.json
data/tutorial.grammar
data/tutorial.examples.json

47
release-core.files
View File

@ -1,47 +0,0 @@
# This file contains all the basic data and software dependencies on
# which the public code release relies to compile, build, and run in
# general.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies core
#
############################################################
# models / data
lib/stanford-corenlp-3.2.0-models.jar
lib/joda-time.jar
lib/jollyday.jar
lib/stanford-corenlp-caseless-2013-06-07-models.jar
# Freebase schema
lib/fb_data/93.exec/schema.ttl
############################################################
# 3rd-party libraries: needed to compile and run
# Fig
lib/fig.jar
# Google libraries
lib/guava-14.0.1.jar
# Json
lib/jackson-core-2.2.0.jar
lib/jackson-annotations-2.2.0.jar
lib/jackson-databind-2.2.0.jar
# Testing
lib/testng-6.8.5.jar
lib/jcommander-1.30.jar
# Lucene
lib/lucene-core-4.4.0.jar
lib/lucene-analyzers-common-4.4.0.jar
lib/lucene-queryparser-4.4.0.jar
# Stanford CoreNLP
lib/stanford-corenlp-3.2.0.jar

40
release-emnlp2013.files
View File

@ -1,40 +0,0 @@
# This file contains additional public datasets described in the EMNLP
# 2013 paper, including WebQuestions.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies emnlp2013
#
############################################################
# Lexicon alignment data
lib/fb_data/6/unaryInfoStringAndAlignment.txt
lib/fb_data/6/binaryInfoStringAndAlignment.txt
############################################################
# WebQuestions
lib/data/webquestions/dataset_11/webquestions.examples.train.json
lib/data/webquestions/dataset_11/webquestions.examples.test.json
############################################################
# Free917 [Cai & Yates, 2013]
data/free917.train.examples.canonicalized.json
data/free917.test.examples.canonicalized.json
############################################################
# Lucene
lib/lucene/4.4/free917
lib/lucene/4.4/inexact
############################################################
# Model
lib/models/15.exec
############################################################
# Grammars
data/emnlp2013.exactent.grammar
data/emnlp2013.inexactent.grammar

9
release-fullfreebase.files
View File

@ -1,9 +0,0 @@
# This file contains a raw ttl file and a Virtuoso index for all of Freebase.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies fullfreebase
#
lib/freebase/93.exec

9
release-fullfreebase_ttl.files
View File

@ -1,9 +0,0 @@
# This file contains a raw ttl file for all of Freebase
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies fullfreebase_ttl
#
lib/freebase/93.exec/0.ttl

9
release-fullfreebase_vdb.files
View File

@ -1,9 +0,0 @@
# This file contains Virtuoso index for all of Freebase.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies fullfreebase_vdb
#
lib/freebase/93.exec/vdb

10
release-geofreebase.files
View File

@ -1,10 +0,0 @@
# This file contains a raw ttl file and a Virtuoso index for a geographic
# subset of Freebase.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies geofreebase
#
lib/freebase/98.exec

8
release-geofreebase_ttl.files
View File

@ -1,8 +0,0 @@
# This file contains a raw ttl file for a geographic subset of Freebase.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies geofreebase_ttl
#
lib/freebase/98.exec/0.ttl

8
release-geofreebase_vdb.files
View File

@ -1,8 +0,0 @@
# This file contains a Virtuoso index for a geographic subset of Freebase.
#
# These files are hosted on the Stanford NLP web endpoint and are
# intended to be obtained via the command
#
# ./download-dependencies geofreebase_vdb
#
lib/freebase/98.exec/vdb

426
run Executable file
View File

@ -0,0 +1,426 @@
#!/usr/bin/ruby
# This is the main entry point for running all SEMPRE programs. See
# fig/lib/execrunner.rb for more documentation for how commands are generated.
# There are a bunch of modes that this script can be invoked with, which
# loosely correspond to the modules.
$: << 'fig/lib'
require 'execrunner'
$modes = []
def addMode(name, description, func)
$modes << [name, description, func]
end
def header(modules='core')
l(
# Queuing system
letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
# Create execution directory
'fig/bin/qcreate',
# Run the Java command...
'java',
'-ea',
'-Dmodules='+modules,
'-Xmx10g',
'-cp', 'libsempre/*:lib/*',
# Profiling
letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'),
nil)
end
def rlwrap; system('which rlwrap') ? 'rlwrap' : nil end
def unbalancedTrainDevSplit
l(o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2))
end
def balancedTrainDevSplit
l(o('Dataset.trainFrac', 0.5), o('Dataset.devFrac', 0.5))
end
def figOpts; l(o('execDir', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end
############################################################
# Unit tests
addMode('test', 'Run unit tests', lambda { |e|
l(
'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*', 'org.testng.TestNG',
lambda { |e|
if e[:class]
l('-testclass', 'edu.stanford.nlp.sempre.' + e[:class])
else
'testng.xml'
end
},
lambda { |e|
if e[:fast]
o('excludegroups', 'sparql,corenlp')
else
nil
end
},
nil)
})
############################################################
# Freebase
def freebaseHeader; header('core,freebase') end
def freebaseFeatureDomains
[
'basicStats',
'alignmentScores',
'entityFeatures',
'context',
'skipPos',
'joinPos',
'wordSim',
'lexAlign',
'tokenMatch',
'rule',
'opCount',
'constant',
'denotation',
'whType',
'span',
'derivRank',
'lemmaAndBinaries',
nil].compact
end
def sparqlOpts
l(
required(:sparqlserver, 'host:port of the Sparql server'), # Example: jonsson:3093, etc.
o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}),
nil)
end
def freebaseOpts
l(
figOpts,
sparqlOpts,
# Features
o('FeatureExtractor.featureDomains', *freebaseFeatureDomains),
o('Builder.executor', 'freebase.SparqlExecutor'),
o('Builder.valueEvaluator', 'freebase.FreebaseValueEvaluator'),
o('LanguageAnalyzer.languageAnalyzer', 'corenlp.CoreNLPAnalyzer'),
# Lexicon
o('LexiconFn.lexiconClassName', 'edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon'),
l( # binary
o('BinaryLexicon.binaryLexiconFilesPath', 'lib/fb_data/7/binaryInfoStringAndAlignment.txt'),
o('BinaryLexicon.keyToSortBy', 'Intersection_size_typed'),
nil),
o('UnaryLexicon.unaryLexiconFilePath', 'lib/fb_data/7/unaryInfoStringAndAlignment.txt'), # unary
o('EntityLexicon.entityPopularityPath', 'lib/fb_data/7/entityPopularity.txt'), # entity
nil)
end
def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath)
l(
required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or <hostname>:<port> (hit the cacheserver)'),
lambda { |e|
cacheserver = e[:cacheserver]
cacheserver = 'jonsson:4000' if cacheserver == 'remote' # Default
case cacheserver
when 'none' then l()
when 'local' then l( # Use files directly - don't run more than one job that does this!
o('Lexicon.cachePath', 'LexiconFn.cache'),
o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'),
nil)
else l(
o('Lexicon.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+lexiconFnCachePath),
o('SparqlExecutor.cachePath', cacheserver+':/u/nlp/data/semparse/cache/'+sparqlExecutorCachePath),
nil)
end
},
nil)
end
# tag is either "free917" or "webquestions"
def emnlp2013AblationExperiments(tag)
l(
letDefault(:ablation, 0),
# Ablation experiments (EMNLP)
sel(:ablation,
l(), # (0) Just run things normally
selo(nil, 'Parser.beamSize', 200, 50, 10), # (1) Vary beam size
selo(nil, 'Dataset.trainFrac', 0.1, 0.2, 0.4, 0.6), # (2) Vary training set size
sel(nil, # (3) Structural: only do join or only do bridge
o('Grammar.tags', l(tag, 'join')),
o('Grammar.tags', l(tag, 'bridge')),
o('Grammar.tags', l(tag, 'inject')),
nil),
sel(nil, # (4) Features
o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains+['lexAlign'])), # +lexAlign
o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains+['lexAlign']-['alignmentScores'])), # +lexAlign -alignmentScores
o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains-['denotation'])), # -denotation
o('FeatureExtractor.featureDomains', *(freebaseFeatureDomains-['skipPos', 'joinPos'])), # -syntax features (skipPos, joinPos)
nil),
#o('Builder.executor', 'FormulaMatchExecutor'), # (6) train on logical forms (doesn't really work well)
nil),
letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1, 2, 3),
nil)
end
def free917
l( # Data
letDefault(:data, 0),
sel(:data,
l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2
l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json')), # (1) Don't run on test yet!
nil),
# Grammar
o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'),
o('Parser.beamSize', 500),
emnlp2013AblationExperiments('free917'),
# lexicon index
letDefault(:lucene, 0),
sel(:lucene,
l(
o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'),
cachePaths('10/LexiconFn.cache', '10/SparqlExecutor.cache'),
o('Grammar.tags', 'free917', 'bridge', 'join', 'inject', 'exact'),
nil),
l( # With entity disambiguation - currently too crappy
o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
cachePaths('4/LexiconFn.cache', '4/SparqlExecutor.cache'),
o('Grammar.tags', 'free917', 'bridge', 'join', 'inject', 'inexact'),
nil),
nil),
# Use binary predicate features (overfits on free917)
o('BridgeFn.filterBadDomain',false),
# Learning
o('Learner.maxTrainIters', 6),
nil)
end
def webquestions
l(
# Data
letDefault(:data, 0),
sel(:data,
l( # Webquestions (dev) [EMNLP final JSON]
o('Dataset.inPaths',
'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'),
unbalancedTrainDevSplit,
nil),
l( # Webquestions (test) [EMNLP final JSON]
o('Dataset.inPaths',
'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json',
'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'),
nil),
nil),
# Grammar
o('Grammar.inPaths', 'freebase/data/emnlp2013.grammar'),
o('Parser.beamSize', 200), # {07/03/13}: WebQuestions is too slow to run with default 500, so set to 200 for now...
# Caching
letDefault(:entitysearch, 1),
sel(:entitysearch, # Used for EMNLP 2013
l(
cachePaths('lucene/0.cache', 'sparql/1.cache'),
o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
o('LexiconFn.maxEntityEntries',10),
o('Grammar.tags', 'webquestions', 'bridge', 'join', 'inject','inexact'), # specify also strategy
nil),
nil),
# Learning
o('Learner.maxTrainIters', 3),
# Use binary predicate features (overfits on free917)
o('BridgeFn.useBinaryPredicateFeatures', true),
o('BridgeFn.filterBadDomain',true),
letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1,2,3),
nil)
end
addMode('freebase', 'Freebase (for EMNLP 2013, ACL 2014, TACL 2014)', lambda { |e| l(
letDefault(:train, 0),
letDefault(:interact, 0),
sel(:interact, l(), rlwrap),
freebaseHeader,
'edu.stanford.nlp.sempre.Main',
freebaseOpts,
# Dataset
sel(:domain, {
'webquestions' => webquestions,
'free917' => free917,
}),
# Training
sel(:train, l(), l(
letDefault(:agenda, 0),
sel(:agenda, l(), agendaExperiments, agendaFree917Experiments),
nil)),
sel(:interact, l(), l(
# After training, run interact, which loads up a set of parameters and
# puts you in a prompt.
o('Dataset.inPaths'),
o('Learner.maxTrainIters', 0),
required(:load, 'none or exec number (e.g., 15) to load'),
lambda { |e|
if e[:load] == 'none' then
l()
else
execPath = "lib/models/#{e[:load]}.exec"
l(
o('Builder.inParamsPath', execPath+'/params'),
o('Grammar.inPaths', execPath+'/grammar'),
o('Master.logPath', lambda{|e| 'state/' + e[:domain] + '.log'}),
o('Master.newExamplesPath', lambda{|e| 'state/' + e[:domain] + '.examples'}),
o('Master.onlineLearnExamples', true),
# Make sure features are set properly!
nil)
end
},
o('Main.interactive'),
nil))
) })
addMode('cacheserver', 'Start the general-purpose cache server that serves files with key-value maps', lambda { |e|
l(
'java', '-Xmx36g', '-ea', '-cp', 'libsempre/*:lib/fig.jar',
'edu.stanford.nlp.sempre.cache.StringCacheServer',
letDefault(:port, 4000),
lambda { |e| o('port', e[:port]) },
letDefault(:cachetype, 1),
sel(:cachetype,
l(
o('FileStringCache.appendMode'),
o('FileStringCache.capacity', 35 * 1024),
o('FileStringCache.flushFrequency', 2147483647),
nil),
l(
o('FileStringCache.appendMode',false),
o('FileStringCache.capacity', 1 * 1024),
o('FileStringCache.flushFrequency', 100000),
nil),
nil),
nil)
})
############################################################
# Freebase RDF database (for building SPARQL database)
# Scratch directory
def scrOptions
letDefault(:scr, '/u/nlp/data/semparse/rdf/scr/' + `hostname | cut -f 1 -d .`.chomp)
end
addMode('filterfreebase', '(1) Filter RDF Freebase dump (do this once) [takes about 1 hour]', lambda { |e| l(
scrOptions,
l(
'fig/bin/qcreate', o('statePath', lambda{|e| e[:scr] + '/state'}),
'java', '-ea', '-Xmx20g', '-cp', 'libsempre/*:lib/*',
'edu.stanford.nlp.sempre.freebase.FilterFreebase',
o('inPath', '/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized'),
sel(:keep, {
'all' => o('keepAllProperties'),
'geo' => l(
o('keepTypesPaths', 'data/geo.types'),
o('keepPropertiesPath', 'data/geo.properties'),
o('keepGeneralPropertiesOnlyForSeenEntities', true),
nil),
}),
o('execDir', '_OUTPATH_'), o('overwriteExecDir'),
nil),
nil) })
addMode('sparqlserver', '(2) Start the SPARQL server [do this every time]', lambda { |e| l(
scrOptions,
required(:exec),
'freebase/scripts/virtuoso', 'start',
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
lambda{|e| 3000+e[:exec]}, # port
nil) })
# (3) Index the filtered RDF dump [takes 48 hours]
addMode('indexfreebase', '(3) Index the filtered RDF dump [takes 48 hours for Freebase]', lambda { |e| l(
letDefault(:stage, nil),
scrOptions,
required(:exec),
sel(:stage,
l(
'scripts/virtuoso', 'add',
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/0.ttl'}, # ttl file
lambda{|e| 3000+e[:exec]}, # port
lambda{|e| e[:offset] || 0}, # offset
nil),
l(
'scripts/extract-freebase-schema.rb',
lambda{|e| 'http://localhost:'+(3000+e[:exec]).to_s+'/sparql'}, # port
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/schema.ttl'},
nil),
nil),
nil) })
addMode('convertfree917', 'Convert the Free917 dataset', lambda { |e| l(
'java', '-ea', '-Xmx15g',
'-cp', 'libsempre/*:lib/*',
'edu.stanford.nlp.sempre.freebase.Free917Converter',
o('inDir','/u/nlp/data/semparse/yates/final-dataset-acl-2013-all/'),
o('outDir','data/free917_convert/'),
o('entityInfoFile','/user/joberant/scr/fb_data/3/entityInfo.txt'),
o('cvtFile','lib/fb_data/2/Cvts.txt'),
o('midToIdFile','/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map'),
nil) })
addMode('query', 'Query a single logical form or SPARQL', lambda { |e| l(
'java', '-ea',
'-cp', 'libsempre/*:lib/*',
'edu.stanford.nlp.sempre.freebase.SparqlExecutor',
sparqlOpts,
nil) })
############################################################
# Just start a simple interactive shell to try out SEMPRE commands
addMode('simple', 'Simple shell', lambda { |e| l(
rlwrap, 'java', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
o('interactive'),
nil) })
addMode('simple-sparql', 'Simple shell for querying SPARQL', lambda { |e| l(
rlwrap, 'java', '-Dmodules=core,freebase', '-cp', 'libsempre/*:lib/*', '-ea', 'edu.stanford.nlp.sempre.Main',
o('executor', 'freebase.SparqlExecutor'),
sparqlOpts,
o('interactive'),
nil) })
############################################################
if ARGV.size == 0
puts "#{$0} @mode=<mode> [options]"
puts
puts 'This is the main entry point for all SEMPRE-related runs.'
puts "Modes:"
$modes.each { |name,description,func|
puts " #{name}: #{description}"
}
end
modesMap = {}
$modes.each { |name,description,func|
modesMap[name] = func
}
run!(sel(:mode, modesMap))

7
scripts/agenda-stats Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/ruby
ARGV.each { |e|
e = e.sub(/\.exec$/, '')
puts "===== #{e}"
system "./fig/bin/tab -s -H -i e/#{e}.exec/learner.events -a iter group .sort utterance / ^parseTime$ ^numOfFeaturizedDerivs$ ^firstCorrectItem$ ^totalDerivs$ ^partCorrect$ ^partOracle$"
}

10
scripts/checkstyle.sh Executable file
View File

@ -0,0 +1,10 @@
# Check style of the code
if [ -z "$1" ]; then
files=`find src -name "*.java"`
else
files="$@"
fi
d=`dirname $0`
java -cp $d/../lib/checkstyle/checkstyle-6.1.1-all.jar com.puppycrawl.tools.checkstyle.Main -c `dirname $0`/checkstyle.xml $files

197
scripts/checkstyle.xml Normal file
View File

@ -0,0 +1,197 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<!--
Checkstyle configuration that checks the sun coding conventions from:
- the Java Language Specification at
http://java.sun.com/docs/books/jls/second_edition/html/index.html
- the Sun Code Conventions at http://java.sun.com/docs/codeconv/
- the Javadoc guidelines at
http://java.sun.com/j2se/javadoc/writingdoccomments/index.html
- the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html
- some best practices
Checkstyle is very configurable. Be sure to read the documentation at
http://checkstyle.sf.net (or in your downloaded distribution).
Most Checks are configurable, be sure to consult the documentation.
To completely disable a check, just comment it out or delete it from the file.
Finally, it is worth reading the documentation.
-->
<module name="Checker">
<!--
If you set the basedir property below, then all reported file
names will be relative to the specified directory. See
http://checkstyle.sourceforge.net/5.x/config.html#Checker
<property name="basedir" value="${basedir}"/>
-->
<!-- Checks that a package-info.java file exists for each package. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html#JavadocPackage -->
<!-- RELAX -->
<!--<module name="JavadocPackage"/>-->
<!-- Checks whether files end with a new line. -->
<!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
<module name="NewlineAtEndOfFile"/>
<!-- Checks that property files contain the same keys. -->
<!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
<module name="Translation"/>
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="FileLength"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="FileTabCharacter"/>
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="RegexpSingleline">
<property name="format" value="\s+$"/>
<property name="minimum" value="0"/>
<property name="maximum" value="0"/>
<property name="message" value="Line has trailing spaces."/>
</module>
<!-- Checks for Headers -->
<!-- See http://checkstyle.sf.net/config_header.html -->
<!-- <module name="Header"> -->
<!-- <property name="headerFile" value="${checkstyle.header.file}"/> -->
<!-- <property name="fileExtensions" value="java"/> -->
<!-- </module> -->
<module name="TreeWalker">
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<!-- RELAX -->
<!--<module name="JavadocMethod"/>
<module name="JavadocType"/>
<module name="JavadocVariable"/>
<module name="JavadocStyle"/>-->
<!-- Checks for Naming Conventions. -->
<!-- See http://checkstyle.sf.net/config_naming.html -->
<!-- RELAX: Rule.rootCat -->
<!--<module name="ConstantName"/>-->
<module name="LocalFinalVariableName"/>
<module name="LocalVariableName"/>
<module name="MemberName"/>
<module name="MethodName">
<property name="format" value="^[a-zA-Z][a-zA-Z0-9]*$"/>
</module>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="TypeName"/>
<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
<!-- RELAX -->
<!--<module name="AvoidStarImport"/>-->
<module name="IllegalImport"/> <!-- defaults to sun.* packages -->
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<!-- RELAX -->
<!--<module name="LineLength"/>-->
<!-- RELAX -->
<!--<module name="MethodLength"/>-->
<!-- RELAX -->
<!--<module name="ParameterNumber"/>-->
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="EmptyForIteratorPad"/>
<module name="GenericWhitespace"/>
<module name="MethodParamPad"/>
<module name="NoWhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<!-- RELAX: allow line to end with || -->
<!--<module name="OperatorWrap"/>-->
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="WhitespaceAfter"/>
<module name="WhitespaceAround"/>
<!-- Modifier Checks -->
<!-- See http://checkstyle.sf.net/config_modifiers.html -->
<module name="ModifierOrder"/>
<module name="RedundantModifier"/>
<!-- Checks for blocks. You know, those {}'s -->
<!-- See http://checkstyle.sf.net/config_blocks.html -->
<!-- RELAX -->
<!--<module name="AvoidNestedBlocks"/>-->
<!--<module name="EmptyBlock"/>-->
<!--<module name="LeftCurly"/>-->
<!--<module name="NeedBraces"/>-->
<!--<module name="RightCurly"/>-->
<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
<!-- RELAX -->
<!--<module name="AvoidInlineConditionals"/>-->
<module name="EmptyStatement"/>
<module name="EqualsHashCode"/>
<!-- RELAX: allow this.x = x in constructors -->
<!--<module name="HiddenField"/>-->
<module name="IllegalInstantiation"/>
<!-- RELAX -->
<!--<module name="InnerAssignment"/>-->
<!-- RELAX: allow numbers -->
<!--<module name="MagicNumber"/>-->
<module name="MissingSwitchDefault"/>
<module name="RedundantThrows">
<property name="suppressLoadErrors" value="true"/>
</module>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>
<!-- Checks for class design -->
<!-- See http://checkstyle.sf.net/config_design.html -->
<!-- RELAX -->
<!--<module name="DesignForExtension"/>-->
<module name="FinalClass"/>
<module name="HideUtilityClassConstructor"/>
<module name="InterfaceIsType"/>
<!-- RELAX: so fig options work -->
<!--<module name="VisibilityModifier"/>-->
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="ArrayTypeStyle"/>
<!-- RELAX -->
<!--<module name="FinalParameters"/>-->
<module name="TodoComment"/>
<module name="UpperEll"/>
</module>
</module>

67
scripts/evaluation.py Executable file
View File

@ -0,0 +1,67 @@
#!/usr/bin/python
import sys
import json
# Official evaluation script used to evaluate Freebase question answering
# systems. Used for EMNLP 2013, ACL 2014 papers, etc.
if len(sys.argv) != 2:
sys.exit("Usage: %s <result_file>" % sys.argv[0])
"""return a tuple with recall, precision, and f1 for one example"""
def computeF1(goldList,predictedList):
"""Assume all questions have at least one answer"""
if len(goldList)==0:
raise Exception("gold list may not be empty")
"""If we return an empty list recall is zero and precision is one"""
if len(predictedList)==0:
return (0,1,0)
"""It is guaranteed now that both lists are not empty"""
precision = 0
for entity in predictedList:
if entity in goldList:
precision+=1
precision = float(precision) / len(predictedList)
recall=0
for entity in goldList:
if entity in predictedList:
recall+=1
recall = float(recall) / len(goldList)
f1 = 0
if precision+recall>0:
f1 = 2*recall*precision / (precision + recall)
return (recall,precision,f1)
averageRecall=0
averagePrecision=0
averageF1=0
count=0
"""Go over all lines and compute recall, precision and F1"""
with open(sys.argv[1]) as f:
for line in f:
tokens = line.split("\t")
gold = json.loads(tokens[1])
predicted = json.loads(tokens[2])
recall, precision, f1 = computeF1(gold,predicted)
averageRecall += recall
averagePrecision += precision
averageF1 += f1
count+=1
"""Print final results"""
averageRecall = float(averageRecall) / count
averagePrecision = float(averagePrecision) / count
averageF1 = float(averageF1) / count
print "Number of questions: " + str(count)
print "Average recall over questions: " + str(averageRecall)
print "Average precision over questions: " + str(averagePrecision)
print "Average f1 over questions: " + str(averageF1)
averageNewF1 = 2 * averageRecall * averagePrecision / (averagePrecision + averageRecall)
print "F1 of average recall and average precision: " + str(averageNewF1)

24
scripts/extract-module-classes.rb Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/ruby
# Input: src
# Output: module-classes.txt
# For each module (e.g., core, freebase), we compute the list of classes
# associated with that class.
out_path = 'module-classes.txt'
out = open(out_path, 'w')
items = []
core_packages = ['test'] # Packages in core which are not their own modules
modules = {}
Dir['src/**/*.java'].each { |path|
class_name = path.sub(/^src\//, '').gsub(/\//, '.').gsub(/\.java$/, '')
module_name = path.sub(/^.*sempre\//, '').split(/\//)[0]
module_name = 'core' if module_name =~ /\.java$/ || core_packages.index(module_name)
modules[module_name] = true
items << (module_name + " " + class_name)
}
items.sort!
out.puts items
out.close
puts "Wrote modules with #{items.size} files to #{out_path}: #{modules.keys.sort.join(' ')}"

10
scripts/find-first-pred-diff.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/sh
# Takes two log files and outputs the first line where they differ in terms
# of the example or the predictions
egrep "Example: |Pred@| Item " ../state/execs/$1.exec/log > temp.1
egrep "Example: |Pred@| Item " ../state/execs/$2.exec/log > temp.2
cmp temp.1 temp.2
#rm temp.1
#rm temp.2

19
scripts/find-hard-coded-paths.rb Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/ruby
# Heuristically find all hard-coded paths in the source code.
# There should be no absolute paths.
Dir["src/**/*.java"].each { |sourcePath|
IO.foreach(sourcePath) { |line|
next unless line =~ /"([\w_\/\.]+)"/
file = $1
next unless file =~ /^\/[uU]\w+\/\w+/ || file =~ /^lib\//
if file =~ /^\//
message = " [BAD: absolute path]"
elsif not File.exists?(file)
message = " [BAD: does not exist]"
else
message = ""
end
puts sourcePath + ": " + file + message
}
}

113
scripts/fix-checkstyle.rb Executable file
View File

@ -0,0 +1,113 @@
#!/usr/bin/ruby
# Hacky script for automatically fixing style errors. This script is far from
# perfect and you should manually inspect all changes before making changes.
# Usage:
# scripts/fix-checkstyle.rb # Inspect changes
# scripts/fix-checkstyle.rb mutate # Apply
# scripts/fix-checkstyle.rb <files> # Do to particular files
mutate = ARGV.index('mutate'); ARGV = ARGV.select { |a| a != 'mutate' }
files = ARGV.size > 0 ? ARGV : Dir['src/**/*.java']
files.each { |path|
in_multi_comment = false
line_num = 0
lines = IO.readlines(path).map { |line|
line_num += 1
line = line.chomp
new_line = line + ''
# Remove trailing whitespace
new_line.gsub!(/\s+$/, '')
# Add space after comment
new_line.gsub!(/ \/\/(\w)/, '// \1')
# Put space after certain operators
new_line.gsub!(/ (if|for|while|catch)\(/, ' \1 (')
# Put space
new_line.gsub!(/( for \(\w+ \w+): /, '\1 : ')
# Put space after casts
new_line.gsub!(/\((\w+)\)([a-z])/, '(\1) \2')
# Reverse modifier order
new_line.gsub!(/ final static /, ' static final ')
in_single_quote = false
in_double_quote = false
in_comment = false
tokens = new_line.split(//)
(0...tokens.size).each { |i|
# Previous, current, next characters
p = i-1 >= 0 ? tokens[i-1] : ''
c = tokens[i]
n = i+1 < tokens.size ? tokens[i+1] : ''
if c == '\'' && p != '\\' # Quote
in_single_quote = !in_single_quote
end
if c == '"' && p != '\\' # Quote
in_double_quote = !in_double_quote
end
if c == '/' && n == '/' # Comment
in_comment = true
end
if c == '/' && n == '*' # Begin multi-comment
in_multi_comment = true
end
if !(in_single_quote || in_double_quote || in_comment || in_multi_comment || c == '')
# Replace if not in quote or comment
if c == ',' # One space after
c += ' ' if n != ' '
elsif ['++', '--'].index(c+n) # Double character operators
c = c+n
n = ''
elsif ['==', '!=', '<=', '>=', '+=', '-=', '*=', '/=', '&=', '|=', '&&', '||'].index(c+n) # Double character operators
c = c+n
n = ''
c = ' ' + c if p != ' '
c = c + ' ' if tokens[i+2] != ' '
elsif '*/=%'.index(c) # Single character operators
# Don't do <, > because of generics
# Don't do , + because of unaries
c = ' ' + c if p != ' '
c = c + ' ' if n != ' '
end
# Write back
tokens[i] = c
tokens[i+1] = n if i+1 < tokens.size
end
if p == '*' && c == '/' # End multi-comment
in_multi_comment = false
end
}
new_line = tokens.join('')
#p new_line
new_line.gsub!(/\. \* ;/, '.*;')
new_line.gsub!(/\s+$/, '')
if line != new_line
puts "======= #{path} #{line_num}"
puts "OLD: [#{line}]"
puts "NEW: [#{new_line}]"
end
new_line
}
# Write it out
if mutate
out = open(path, 'w')
out.puts lines
out.close
end
}

38
scripts/tunnel Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
# Allows running things locally that need to connect to cache servers or sparql
# servers on NLP, which are behind a firewall.
# After starting this script, connect to localhost:<port> to connect to <NLP machine>:<port>.
# Cache server
host=jonsson
port=4000
echo "Tunnel localhost:$port => $host:$port"
pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}')
if [ -n "$pid" ]; then kill $pid; fi
ssh -N -n -L $port:$host:$port jacob.stanford.edu &
# Sparql server for Freebase
host=jonsson
port=3093
echo "Tunnel localhost:$port => $host:$port"
pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}')
if [ -n "$pid" ]; then kill $pid; fi
ssh -N -n -L $port:$host:$port jacob.stanford.edu &
# Sparql server for Paleo
host=jonsson
port=3021
echo "Tunnel localhost:$port => $host:$port"
pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}')
if [ -n "$pid" ]; then kill $pid; fi
ssh -N -n -L $port:$host:$port jacob.stanford.edu &
# Cache server (immutable)
host=john5
port=4001
echo "Tunnel localhost:$port => $host:$port"
pid=$(ps ax | grep ssh.*:$port | grep -v grep | awk '{print $1}')
if [ -n "$pid" ]; then kill $pid; fi
ssh -N -n -L $port:$host:$port jacob.stanford.edu &

107
scripts/verify-code-loop.rb Executable file
View File

@ -0,0 +1,107 @@
#!/usr/bin/ruby
# Verifies that the codebase is sane (compiles, doesn't crash, gets reasonable
# accuracy) every once in a while. If something fails, an email is sent out
$mode = ARGV[0]
if $mode != 'now' && $mode != 'loop'
puts "Usage: #{$0} (now|loop)"
puts " now: check immediately and exit (don't email)"
puts " loop: check only when stuff changes and loop (email if something breaks)"
exit 1
end
# Who should be notified if the code breaks.
$recipient = 'stanford-sempre@googlegroups.com'
$logPath = "verify-code-loop.log"
# Send out the log file to all the recipients.
def emailLog(subject)
return unless $mode == 'loop'
maxLines = 100 # Maximum number of lines to send via email.
numLines = IO.readlines($logPath).size
if numLines <= maxLines
command = "cat #{$logPath}"
else
# Take first few lines and last few lines to keep under maxLines.
command = "(head -#{maxLines/2} #{$logPath}; echo '... (#{numLines - maxLines} lines omitted) ...'; tail -#{maxLines/2} #{$logPath})"
end
command = "#{command} | mail -s '#{subject}' #{$recipient}"
puts "Emailing log file: #{command}"
system command or exit 1
end
def emailBroken
emailLog('sempre code is broken!')
end
# Print to stdout and log file.
def log(line, newline=true)
line = "[#{`date`.chomp}] #{line}"
if newline
puts line
else
print line
end
out = open($logPath, 'a')
out.puts line
out.close
end
# Run and command; if fail, send email.
def run(command, verbose)
log("======== Running: #{command}", false) if verbose
ok = system "#{command} >> #{$logPath} 2>&1"
puts " [#{ok ? 'ok' : 'failed'}]" if verbose
emailBroken if not ok
ok
end
def restart
exit 1 if $mode == 'now'
system "cat #{$logPath}"
# In case there are updates
log("Restarting #{$0}...")
exec($0 + ' loop')
end
log("Started verify-code loop version 2")
log("Writing to #{$logPath}...")
firstTime = true
while true
break if $mode == 'now' && (not firstTime)
# Whenever there's a change to the repository, run a test
system "rm -f #{$logPath}"
if not run('git pull', false)
log("git pull failed - this is bad, let's just quit.")
break
end
if $mode == 'loop' && system("grep -q 'Already up-to-date' #{$logPath}")
# No changes, just wait
sleep 60
next
end
firstTime = false
# Check everything
log("Testing...")
run('git log -3', true) or restart # Print out last commit messages
run('./pull-dependencies', true) or restart
run('make clean', true) or restart
run('make', true) or restart
run('scripts/find-hard-coded-paths.rb', true) or restart
# Run tests
run("./run @mode=test", true) or restart
emailLog('sempre code passes tests!')
break if $mode == 'now'
restart
end

432
sempre
View File

@ -1,432 +0,0 @@
#!/usr/bin/ruby
$: << 'fig/lib'
require 'execrunner'
# Note: run this on the NLP machines because SPARQL server is running on jack.
if ARGV.size == 0
puts "Usage:"
puts " ./sempre @mode=train [additional options] # Train semantic parser"
puts " ./sempre @mode=interact [additional options] # Interactive mode for testing"
puts
puts " ./sempre @mode=cacheserver [additional options] # Start cache server"
puts " ./sempre @mode=sparqlserver [additional options] # Start sparql server"
puts
puts " ./sempre @mode=filterfreebase [additional options] # Filter raw Freebase RDF dump"
puts " ./sempre @mode=indexfreebase [additional options] # Build Freebase index"
puts
puts " ./sempre @mode=test [additional options] # Run TestNG test suite"
puts
puts "Additional options can be any of the following:"
puts " - Additional program options (e.g., -BeamParser.beamSize 3)"
puts " - Execrunner options, which select the options to include (@data=0)"
exit 1
end
system "mkdir -p state/execs"
system "touch state/lastExec"
def header
l(
letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
'fig/bin/qcreate',
(File.exists?('/u/nlp/bin/java7') ? '/u/nlp/bin/java7' : 'java'),
'-ea',
'-Xmx15g',
'-cp', 'classes:'+Dir['lib/*.jar'].join(':'),
letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'),
nil)
end
def testHeader
l(
'java',
'-ea',
'-Xmx12g',
'-cp', 'classes:'+Dir['lib/*.jar'].join(':'),
'org.testng.TestNG',
nil)
end
$lexdir=6
def sparqlOpts
l(
required(:sparqlserver, 'host:port of the Sparql server'),
o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}),
nil)
end
$defaultFeatureDomains = [
'basicStats',
'alignmentScores',
'tokensDistance',
'context',
'skipPos',
'joinPos',
#'bridgeBinaryMatch',
'wordSim',
#'lexAlign',
'tokenMatch',
#'rule',
'opCount',
'constant',
'denotation',
'whType',
#'lemmaAndBinaries',
nil].compact
def defaultOpts
l(
o('execDir', '_OUTPATH_'), o('overwriteExecDir'),
o('addToView', 0),
selo(0, 'LexiconFn.lexiconClassName',
'edu.stanford.nlp.sempre.fbalignment.lexicons.Lexicon',
'edu.stanford.nlp.sempre.fbalignment.lexicons.WordNetExpansionLexicon',
'edu.stanford.nlp.sempre.fbalignment.lexicons.GraphPropLexicon'),
letDefault(:executor, 1),
sel(:executor,
l(),
sparqlOpts,
nil),
selo(:executor, 'Builder.executor',
'FormulaMatchExecutor', # Measure accuracy using logical forms (faster); good proxy
'SparqlExecutor', # Measure accuracy using answers (slower); executes SPARQL query on database
nil),
letDefault(:binarylex,0),
sel(:binarylex,
l(
o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/'+$lexdir.to_s+'/binaryInfoStringAndAlignment.txt'),
o('BinaryLexicon.keyToSortBy','Intersection_size_typed'),
nil),
l(
sel(nil,
o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/graphprop/1/jaccardLexicon','lib/fb_data/graphprop/1/graphpropLexicon'),
o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/graphprop/2/jaccardLexicon','lib/fb_data/graphprop/2/graphpropLexicon'),
nil),
o('BinaryLexicon.keyToSortBy','graphprop_estimate'),
nil),
l(
o('BinaryLexicon.binaryLexiconFilesPath','lib/fb_data/graphprop/1/jaccardLexicon'),
o('BinaryLexicon.keyToSortBy','jaccard'),
nil),
nil),
# binary
o('BinaryLexicon.useOnlyJaccard',false),
# unary
o('UnaryLexicon.unaryLexiconFilePath','lib/fb_data/'+$lexdir.to_s+'/unaryInfoStringAndAlignment.txt'),
nil)
end
def unbalancedTrainDevSplit
l(o('Dataset.trainFrac', 0.8), o('Dataset.devFrac', 0.2))
end
def balancedTrainDevSplit
l(o('Dataset.trainFrac', 0.5), o('Dataset.devFrac', 0.5))
end
def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath)
l(
required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or <hostname>:<port> (hit the cacheserver)'),
lambda { |e|
cacheserver = e[:cacheserver]
case cacheserver
when 'none' then l()
when 'local' then l( # Use files directly - don't run more than one job that does this!
o('Lexicon.cachePath', 'LexiconFn.cache'),
o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'),
nil)
else l(
o('Lexicon.cachePath', cacheserver + ':' + lexiconFnCachePath),
o('SparqlExecutor.cachePath', cacheserver + ':' + sparqlExecutorCachePath),
nil)
end
},
nil)
end
# tag is either "free917" or "webquestions"
def emnlp2013AblationExperiments(tag)
l(
letDefault(:ablation, 0),
# Ablation experiments (EMNLP)
sel(:ablation,
l(), # (0) Just run things normally
selo(nil, 'BeamParser.beamSize', 10, 50, 200), # (1) Vary beam size
selo(nil, 'Dataset.trainFrac', 0.1, 0.2, 0.4, 0.6), # (2) Vary training set size
sel(nil, # (3) Structural: only do join or only do bridge
o('Grammar.tags', l(tag, 'join')),
o('Grammar.tags', l(tag, 'bridge')),
nil),
sel(nil, # (4) Features
o('FeatureExtractor.featureDomains', *($defaultFeatureDomains+['lexAlign'])), # +lexAlign
o('FeatureExtractor.featureDomains', *($defaultFeatureDomains+['lexAlign']-['alignmentScores'])), # +lexAlign -alignmentScores
o('FeatureExtractor.featureDomains', *($defaultFeatureDomains-['denotation'])), # -denotation
o('FeatureExtractor.featureDomains', *($defaultFeatureDomains-['skipPos', 'joinPos'])), # -syntax features (skipPos, joinPos)
nil),
nil),
letDefault(:split, 0), selo(:split, 'Dataset.splitRandom', 1, 2, 3),
nil)
end
def free917
l( # Data
letDefault(:data, 0),
sel(:data,
l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json'), unbalancedTrainDevSplit), # (0) train 0.8, dev 0.2
l(o('Dataset.inPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json')), # (1) Careful, this is test set!
nil),
# Grammar
o('Grammar.inPaths', 'data/emnlp2013.inexactent.grammar'),
o('Grammar.tags', 'free917', 'bridge', 'join'),
# Features
o('FeatureExtractor.featureDomains', *$defaultFeatureDomains),
emnlp2013AblationExperiments('free917'),
l(
o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'),
o('Lexicon.entitySearchStrategy','exact'),
cachePaths('LexiconFn.cache', 'SparqlExecutor.cache'),
nil),
nil)
end
def webquestions
l(
# Data
letDefault(:data, 0),
sel(:data,
l( # Webquestions (dev)
o('Dataset.inPaths',
'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'),
unbalancedTrainDevSplit,
nil),
l( # Webquestions (test)
o('Dataset.inPaths',
'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json',
'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'),
nil),
nil),
# Grammar
o('Grammar.inPaths', 'data/emnlp2013.inexactent.grammar'),
o('Grammar.tags', 'webquestions', 'bridge', 'join'),
o('BeamParser.beamSize', 200),
# Features
o('FeatureExtractor.featureDomains', *($defaultFeatureDomains+['lexAlign'])),
emnlp2013AblationExperiments('webquestions'),
# Caching
cachePaths('LexiconFn.cache', 'SparqlExecutor.cache'),
o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
o('EntityLexicon.maxEntries',50),
# Learner reward
o('Learner.partialReward', true),
nil)
end
def jayant
l(
# Data
letDefault(:data, 1),
sel(:data,
l(o('Dataset.inPaths', 'train,data/jayant-emnlp2012-validation.examples.json'), balancedTrainDevSplit),
l(o('Dataset.inPaths', 'train,data/jayant-emnlp2012-validation.examples.json','test,data/jayant-emnlp2012-test.examples.json')),
nil),
# Grammar
o('Grammar.inPaths', 'data/emnlp2013.inexactent.grammar'),
o('Grammar.tags', 'webquestions', 'bridge', 'join'),
o('BeamParser.beamSize', 500),
# Caching
cachePaths('LexiconFn.cache', 'SparqlExecutor.cache'),
o('EntityLexicon.inexactMatchIndex','lib/lucene/4.4/inexact/'),
nil)
end
def selectDomain
l(
letDefault(:domain, 0),
sel(:domain, {
'none' => l(),
'webquestions' => webquestions,
'free917' => free917,
'jayant' => jayant,
}),
nil)
end
def train
l(
header,
'edu.stanford.nlp.sempre.Main',
defaultOpts, selectDomain,
o('Learner.maxTrainIters', 4),
nil)
end
def interact
l(
# After training, run interact, which loads up a set of parameters and
# puts you in a prompt.
system('which rlwrap') ? 'rlwrap' : nil,
header,
'edu.stanford.nlp.sempre.Main',
defaultOpts, selectDomain,
o('Dataset.inPaths'),
o('Learner.maxTrainIters', 0),
required(:load, 'none or exec number (e.g., 812) to load'),
lambda { |e|
if e[:load] == 'none' then
l()
else
execPath = "lib/models/#{e[:load]}.exec"
l(
o('Builder.inParamsPath', execPath+'/params'),
o('Grammar.inPaths', execPath+'/grammar'),
o('Master.logPath', lambda{|e| 'state/' + e[:domain] + '.log'}),
o('Master.newExamplesPath', lambda{|e| 'state/' + e[:domain] + '.examples'}),
o('Master.onlineLearnExamples', true),
# Make sure features are set properly!
nil)
end
},
o('Main.interactive'),
sel(:executeTopOnly,
l(),
l(
o('Parser.executeTopFormulaOnly'),
a('FeatureExtractor.disableDenotationFeatures', true),
nil),
nil),
nil)
end
# Start the cache server that serves files with key-value maps.
def cacheserver
l(
'java', '-Xmx15g', '-ea', '-cp', 'classes:lib/fig.jar',
'edu.stanford.nlp.sempre.StringCacheServer',
o('port', 4000),
nil)
end
############################################################
# Freebase RDF database
# Scratch directory
def scrOptions
letDefault(:scr, '/u/nlp/data/semparse/rdf/scr/' + `hostname | cut -f 1 -d .`.chomp)
end
# (1) Filter RDF Freebase dump (do this once) [takes about 1 hour]
def filterfreebase
l(
scrOptions,
l(
'fig/bin/qcreate', o('statePath', lambda{|e| e[:scr] + '/state'}),
'java', '-ea', '-Xmx20g', '-cp', 'classes:lib/*',
'edu.stanford.nlp.sempre.freebase.FilterFreebase',
o('inPath', '/u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized'),
sel(:keep, {
'all' => o('keepAllProperties'),
'geo' => l(
o('keepTypesPaths', 'data/geo.types'),
o('keepPropertiesPath', 'data/geo.properties'),
o('keepGeneralPropertiesOnlyForSeenEntities', true),
nil),
}),
o('execDir', '_OUTPATH_'), o('overwriteExecDir'),
nil),
nil)
end
# (2) Start the SPARQL server (do this every time).
def sparqlserver
l(
scrOptions,
letDefault(:exec, 93),
'scripts/virtuoso', 'start',
lambda{|e| 'lib/freebase/'+e[:exec].to_s+'.exec/vdb'}, # DB directory
lambda{|e| 3000+e[:exec]}, # port
nil)
end
# (3) Index the filtered RDF dump [takes 48 hours]
# Afterwards, stop the server and copy the exec directory to
# /u/nlp/data/semparse/scr/freebase/state/execs (this will serve as the master
# copy).
def indexfreebase
l(
letDefault(:stage, nil),
scrOptions,
required(:exec),
sel(:stage,
l(
'scripts/virtuoso', 'add',
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/0.ttl'}, # ttl file
lambda{|e| 3000+e[:exec]}, # port
lambda{|e| e[:offset] || 0}, # offset
nil),
l(
'scripts/extract-freebase-schema.rb',
lambda{|e| 'http://localhost:'+(3000+e[:exec]).to_s+'/sparql'}, # port
lambda{|e| e[:scr]+'/state/execs/'+e[:exec].to_s+'.exec/schema.ttl'},
nil),
nil),
nil)
end
# Query a single logical form or SPARQL
def query
l(
'java', '-ea',
'-cp', 'classes:'+Dir['lib/*.jar'].join(':'),
'edu.stanford.nlp.sempre.SparqlExecutor',
sparqlOpts,
nil)
end
def test
l(
testHeader,
lambda { |e|
if e[:class]
l('-testclass', 'edu.stanford.nlp.sempre.test.' + e[:class])
else
'testng.xml'
end
},
nil)
end
run!(
sel(:mode, {
'train' => train,
'interact' => interact,
'cacheserver' => cacheserver,
'filterfreebase' => filterfreebase,
'indexfreebase' => indexfreebase,
'sparqlserver' => sparqlserver,
'query' => query,
'test' => test,
}),
nil)

203
src/edu/stanford/nlp/sempre/AbstractReinforcementParserState.java Normal file
View File

@ -0,0 +1,203 @@
package edu.stanford.nlp.sempre;
import com.google.common.collect.Sets;
import fig.basic.LogInfo;
import fig.basic.MapUtils;
import fig.basic.StopWatchSet;
import java.util.*;
/**
* Contains methods for putting derivations on the chart and combining them
* to add new derivations to the agenda
* @author joberant
*/
abstract class AbstractReinforcementParserState extends ChartParserState {
protected final ReinforcementParser parser;
protected final CoarseParser coarseParser;
protected CoarseParser.CoarseParserState coarseParserState;
protected static final double EPSILON = 10e-20; // used to break ties between agenda items
public AbstractReinforcementParserState(ReinforcementParser parser, Params params, Example ex, boolean computeExpectedCounts) {
super(parser, params, ex, computeExpectedCounts);
this.parser = parser;
coarseParser = parser.coarseParser;
}
protected abstract void addToAgenda(DerivationStream derivationStream);
protected boolean coarseAllows(String cat, int start, int end) {
return coarseParserState == null || coarseParserState.coarseAllows(cat, start, end);
}
//don't add to a cell in the chart that is fill
protected boolean addToBoundedChart(Derivation deriv) {
List<Derivation> derivations = chart[deriv.start][deriv.end].get(deriv.cat);
totalGeneratedDerivs++;
if (Parser.opts.visualizeChartFilling) {
chartFillingList.add(new CatSpan(deriv.start, deriv.end, deriv.cat));
}
if (derivations == null) {
chart[deriv.start][deriv.end].put(deriv.cat,
derivations = new ArrayList<>());
}
if (derivations.size() < getBeamSize()) {
derivations.add(deriv);
Collections.sort(derivations, Derivation.derivScoreComparator); // todo - perhaps can be removed
return true;
} else {
return false;
}
}
// for [start, end) we try to create [start, end + i) or [start - i, end) and add unary rules
protected void combineWithChartDerivations(Derivation deriv) {
expandDerivRightwards(deriv);
expandDerivLeftwards(deriv);
applyCatUnaryRules(deriv);
}
private void expandDerivRightwards(Derivation leftChild) {
if (parser.verbose(6))
LogInfo.begin_track("Expanding rightward");
Map<String, List<Rule>> rhsCategoriesToRules = parser.leftToRightSiblingMap.get(leftChild.cat);
if (rhsCategoriesToRules != null) {
for (int i = 1; leftChild.end + i <= numTokens; ++i) {
Set<String> intersection = Sets.intersection(rhsCategoriesToRules.keySet(), chart[leftChild.end][leftChild.end + i].keySet());
for (String rhsCategory : intersection) {
List<Rule> compatibleRules = rhsCategoriesToRules.get(rhsCategory);
List<Derivation> rightChildren = chart[leftChild.end][leftChild.end + i].get(rhsCategory);
generateParentDerivations(leftChild, rightChildren, true, compatibleRules);
}
}
// handle terminals
if (leftChild.end < numTokens)
handleTerminalExpansion(leftChild, false, rhsCategoriesToRules);
}
if (parser.verbose(6))
LogInfo.end_track();
}
private void expandDerivLeftwards(Derivation rightChild) {
if (parser.verbose(5))
LogInfo.begin_track("Expanding leftward");
Map<String, List<Rule>> lhsCategorisToRules = parser.rightToLeftSiblingMap.get(rightChild.cat);
if (lhsCategorisToRules != null) {
for (int i = 1; rightChild.start - i >= 0; ++i) {
Set<String> intersection = Sets.intersection(lhsCategorisToRules.keySet(), chart[rightChild.start - i][rightChild.start].keySet());
for (String lhsCategory : intersection) {
List<Rule> compatibleRules = lhsCategorisToRules.get(lhsCategory);
List<Derivation> leftChildren = chart[rightChild.start - i][rightChild.start].get(lhsCategory);
generateParentDerivations(rightChild, leftChildren, false, compatibleRules);
}
}
// handle terminals
if (rightChild.start > 0)
handleTerminalExpansion(rightChild, true, lhsCategorisToRules);
}
if (parser.verbose(5))
LogInfo.end_track();
}
private void generateParentDerivations(Derivation expandedDeriv, List<Derivation> otherDerivs,
boolean expandedLeftChild, List<Rule> compatibleRules) {
for (Derivation otherDeriv : otherDerivs) {
Derivation leftChild, rightChild;
if (expandedLeftChild) {
leftChild = expandedDeriv;
rightChild = otherDeriv;
} else {
leftChild = otherDeriv;
rightChild = expandedDeriv;
}
List<Derivation> children = new ArrayList<>();
children.add(leftChild);
children.add(rightChild);
for (Rule rule : compatibleRules) {
if (coarseAllows(rule.lhs, leftChild.start, rightChild.end)) {
DerivationStream resDerivations = applyRule(leftChild.start, rightChild.end, rule, children);
if (!resDerivations.hasNext())
continue;
addToAgenda(resDerivations);
}
}
}
}
// returns the score of derivation computed
private DerivationStream applyRule(int start, int end, Rule rule, List<Derivation> children) {
try {
if (Parser.opts.verbose >= 5)
LogInfo.logs("applyRule %s %s %s %s", start, end, rule, children);
StopWatchSet.begin(rule.getSemRepn()); // measuring time
StopWatchSet.begin(rule.toString());
DerivationStream results = rule.sem.call(ex,
new SemanticFn.CallInfo(rule.lhs, start, end, rule, com.google.common.collect.ImmutableList.copyOf(children)));
StopWatchSet.end();
StopWatchSet.end();
return results;
} catch (Exception e) {
LogInfo.errors("Composition failed: rule = %s, children = %s", rule, children);
e.printStackTrace();
throw new RuntimeException(e);
}
}
private void applyCatUnaryRules(Derivation deriv) {
if (parser.verbose(4))
LogInfo.begin_track("Category unary rules");
for (Rule rule : parser.catUnaryRules) {
if (!coarseAllows(rule.lhs, deriv.start, deriv.end))
continue;
if (deriv.cat.equals(rule.rhs.get(0))) {
DerivationStream resDerivations = applyRule(deriv.start, deriv.end, rule, Collections.singletonList(deriv));
addToAgenda(resDerivations);
}
}
if (parser.verbose(4))
LogInfo.end_track();
}
public List<DerivationStream> gatherRhsTerminalsDerivations() {
List<DerivationStream> derivs = new ArrayList<>();
final List<Derivation> empty = Collections.emptyList();
for (int i = 0; i < numTokens; i++) {
for (int j = i + 1; j <= numTokens; j++) {
for (Rule rule : MapUtils.get(parser.terminalsToRulesList, phrases[i][j], Collections.<Rule>emptyList())) {
if (!coarseAllows(rule.lhs, i, j))
continue;
derivs.add(applyRule(i, j, rule, empty));
}
}
}
return derivs;
}
// rules where one word is a terminal and the other is a non-terminal
private void handleTerminalExpansion(Derivation child, boolean before, Map<String, List<Rule>> categoriesToRules) {
String phrase = before ? phrases[child.start - 1][child.start] : phrases[child.end][child.end + 1];
int start = before ? child.start - 1 : child.start;
int end = before ? child.end : child.end + 1;
if (categoriesToRules.containsKey(phrase)) {
List<Derivation> children = new ArrayList<>();
children.add(child);
for (Rule rule : categoriesToRules.get(phrase)) {
if (coarseAllows(rule.lhs, start, end)) {
DerivationStream resDerivations = applyRule(start, end, rule, children);
if (!resDerivations.hasNext())
continue;
addToAgenda(resDerivations);
}
}
}
}
}

20
src/edu/stanford/nlp/sempre/AggregateFormula.java
View File

@ -3,14 +3,15 @@ package edu.stanford.nlp.sempre;
import com.google.common.base.Function;
import fig.basic.LispTree;
import java.util.List;
/**
* Aggregate takes a set and computes some number of that set.
* Includes existential quantification.
* 'Aggregate' takes a set and computes some number on that set.
*
* @author Percy Liang
*/
public class AggregateFormula extends Formula {
public enum Mode {count, sum, mean, min, max, exists};
public enum Mode { count, sum, avg, min, max };
public final Mode mode;
public final Formula child;
@ -29,10 +30,9 @@ public class AggregateFormula extends Formula {
public static Mode parseMode(String mode) {
if ("count".equals(mode)) return Mode.count;
if ("sum".equals(mode)) return Mode.sum;
if ("mean".equals(mode)) return Mode.mean;
if ("avg".equals(mode)) return Mode.avg;
if ("min".equals(mode)) return Mode.min;
if ("max".equals(mode)) return Mode.max;
if ("exists".equals(mode)) return Mode.max;
return null;
}
@ -41,6 +41,14 @@ public class AggregateFormula extends Formula {
return result == null ? new AggregateFormula(mode, child.map(func)) : result;
}
@Override
public List<Formula> mapToList(Function<Formula, List<Formula>> func, boolean alwaysRecurse) {
List<Formula> res = func.apply(this);
if (res.isEmpty() || alwaysRecurse)
res.addAll(child.mapToList(func, alwaysRecurse));
return res;
}
@Override
public boolean equals(Object thatObj) {
if (!(thatObj instanceof AggregateFormula)) return false;
@ -49,7 +57,7 @@ public class AggregateFormula extends Formula {
if (!this.child.equals(that.child)) return false;
return true;
}
public int computeHashCode() {
int hash = 0x7ed55d16;
hash = hash * 0xd3a2646c + mode.toString().hashCode();

85
src/edu/stanford/nlp/sempre/ArithmeticFormula.java Normal file
View File

@ -0,0 +1,85 @@
package edu.stanford.nlp.sempre;
import com.google.common.base.Function;
import fig.basic.LispTree;
import java.util.List;
/**
* Performs arithmetic operations (+, -, *, /).
* Note that these are non-binary relations, which means we can't model them
* using a join.
*
* @author Percy Liang
*/
public class ArithmeticFormula extends Formula {
public enum Mode { add, sub, mul, div };
public final Mode mode;
public final Formula child1;
public final Formula child2;
public ArithmeticFormula(Mode mode, Formula child1, Formula child2) {
this.mode = mode;
this.child1 = child1;
this.child2 = child2;
}
public LispTree toLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild(modeToString(mode));
tree.addChild(child1.toLispTree());
tree.addChild(child2.toLispTree());
return tree;
}
public Formula map(Function<Formula, Formula> func) {
Formula result = func.apply(this);
return result == null ? new ArithmeticFormula(mode, child1.map(func), child2.map(func)) : result;
}
@Override
public List<Formula> mapToList(Function<Formula, List<Formula>> func, boolean alwaysRecurse) {
List<Formula> res = func.apply(this);
if (res.isEmpty() || alwaysRecurse) {
res.addAll(child1.mapToList(func, alwaysRecurse));
res.addAll(child2.mapToList(func, alwaysRecurse));
}
return res;
}
public static Mode parseMode(String mode) {
if ("+".equals(mode)) return Mode.add;
if ("-".equals(mode)) return Mode.sub;
if ("*".equals(mode)) return Mode.mul;
if ("/".equals(mode)) return Mode.div;
return null;
}
public static String modeToString(Mode mode) {
switch (mode) {
case add: return "+";
case sub: return "-";
case mul: return "*";
case div: return "/";
default: throw new RuntimeException("Invalid mode: " + mode);
}
}
@Override
public boolean equals(Object thatObj) {
if (!(thatObj instanceof ArithmeticFormula)) return false;
ArithmeticFormula that = (ArithmeticFormula) thatObj;
if (this.mode != that.mode) return false;
if (!this.child1.equals(that.child1)) return false;
if (!this.child2.equals(that.child2)) return false;
return true;
}
public int computeHashCode() {
int hash = 0x7ed55d16;
hash = hash * 0xd3a2646c + mode.toString().hashCode(); // Note: don't call hashCode() on mode directly.
hash = hash * 0xd3a2646c + child1.hashCode();
hash = hash * 0xd3a2646c + child2.hashCode();
return hash;
}
}

31
src/edu/stanford/nlp/sempre/AtomicSemType.java Normal file
View File

@ -0,0 +1,31 @@
package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.*;
// Represents an atomic type (strings, entities, numbers, dates, etc.).
public class AtomicSemType extends SemType {
public final String name;
public AtomicSemType(String name) {
if (name == null) throw new RuntimeException("Null name");
this.name = name;
}
public boolean isValid() { return true; }
public SemType meet(SemType that) {
if (that instanceof TopSemType) return this;
if (that instanceof UnionSemType) return that.meet(this);
if (that instanceof AtomicSemType) {
String name1 = this.name;
String name2 = ((AtomicSemType) that).name;
if (name1.equals(name2)) return this; // Shortcut: the same
if (SemTypeHierarchy.singleton.getSupertypes(name1).contains(name2)) return this;
if (SemTypeHierarchy.singleton.getSupertypes(name2).contains(name1)) return that;
return SemType.bottomType;
}
return SemType.bottomType;
}
public SemType apply(SemType that) { return SemType.bottomType; }
public SemType reverse() { return SemType.bottomType; }
public LispTree toLispTree() { return LispTree.proto.newLeaf(name); }
}

21
src/edu/stanford/nlp/sempre/BadFormulaException.java Normal file
View File

@ -0,0 +1,21 @@
package edu.stanford.nlp.sempre;
public class BadFormulaException extends RuntimeException {
public static final long serialVersionUID = 86586128316354597L;
String message;
public BadFormulaException(String message) { this.message = message; }
// Combine multiple exceptions
public BadFormulaException(BadFormulaException... exceptions) {
StringBuilder builder = new StringBuilder();
for (BadFormulaException exception : exceptions)
builder.append(" | ").append(exception.message);
//builder.append(exception).append("\n");
this.message = builder.toString().substring(3);
}
@Override
public String toString() { return message; }
}

279
src/edu/stanford/nlp/sempre/BeamParser.java
View File

@ -4,6 +4,7 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import fig.basic.*;
import fig.exec.Execution;
import java.util.*;
@ -11,46 +12,46 @@ import java.util.*;
* A simple bottom-up chart-based parser that keeps the |beamSize| top
* derivations for each chart cell (cat, start, end). Also supports fast
* indexing of lexicalized rules using a trie.
* <p/>
* In the future, when we have more parsers, some of this code should be
* refactored into Parser.
*
* Note that this code does not rely on the Grammar being binarized,
* which makes it more complex.
*
* @author Percy Liang
*/
public class BeamParser extends Parser {
public static class Options {
@Option public int beamSize = 500;
@Option public int maxNewTreesPerSpan = Integer.MAX_VALUE;
}
public static Options opts = new Options();
Trie trie; //for non-catunary rules
Trie trie; // For non-cat-unary rules
public BeamParser(Spec spec) {
super(spec);
public BeamParser(Grammar grammar, FeatureExtractor extractor, Executor executor) {
super(grammar, extractor, executor);
// Index the non-cat-unary rules
trie = new Trie();
for (Rule rule : grammar.rules)
if (!rule.isCatUnary())
trie.add(rule);
addRule(rule);
if (Parser.opts.visualizeChartFilling)
this.chartFillOut = IOUtils.openOutAppendEasy(Execution.getFile("chartfill"));
}
public int getDefaultBeamSize() {
return BeamParser.opts.beamSize;
public synchronized void addRule(Rule rule) {
if (!rule.isCatUnary())
trie.add(rule);
}
public ParserState newCoarseParserState(Params params, Example ex) {
return new BeamParserState(
ParserState.Mode.bool,
this, params, ex, null);
}
public ParserState newParserState(Params params,
Example ex,
ParserState coarseState) {
return new BeamParserState(
ParserState.Mode.full,
this, params, ex, coarseState);
public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
BeamParserState coarseState = null;
if (Parser.opts.coarsePrune) {
LogInfo.begin_track("Parser.coarsePrune");
coarseState = new BeamParserState(this, params, ex, computeExpectedCounts, BeamParserState.Mode.bool, null);
coarseState.infer();
coarseState.keepTopDownReachable();
LogInfo.end_track();
}
return new BeamParserState(this, params, ex, computeExpectedCounts, BeamParserState.Mode.full, coarseState);
}
}
@ -61,115 +62,98 @@ public class BeamParser extends Parser {
* @author Percy Liang
* @author Roy Frostig
*/
class BeamParserState extends ParserState {
private final BeamParser parser;
class BeamParserState extends ChartParserState {
public final Mode mode;
// Modes:
// 1) Bool: just check if cells (cat, start, end) are reachable (to prune chart)
// 2) Full: compute everything
public enum Mode { bool, full }
public BeamParserState(Mode mode,
BeamParser parser,
Params params,
Example ex,
ParserState coarseState) {
super(mode, parser, params, ex, coarseState);
private final BeamParser parser;
private final BeamParserState coarseState; // Used to prune
public BeamParserState(BeamParser parser, Params params, Example ex, boolean computeExpectedCounts,
Mode mode, BeamParserState coarseState) {
super(parser, params, ex, computeExpectedCounts);
this.parser = parser;
this.mode = mode;
this.coarseState = coarseState;
}
public void infer() {
if (numTokens == 0)
return;
if (parser.verbose(2)) LogInfo.begin_track("ParserState.infer");
// Base case
for (Derivation deriv : gatherTokenAndPhraseDerivations()) {
featurizeAndScoreDerivation(deriv);
addToChart(deriv);
}
// Recursive case
for (int len = 1; len <= numTokens; len++)
for (int i = 0; i + len <= numTokens; i++)
build(i, i + len);
if (parser.verbose(2)) LogInfo.end_track();
// Visualize
if (parser.chartFillOut != null && Parser.opts.visualizeChartFilling && this.mode != Mode.bool) {
parser.chartFillOut.println(Json.writeValueAsStringHard(new ChartFillingData(ex.id, chartFillingList,
ex.utterance, ex.numTokens())));
parser.chartFillOut.flush();
}
setPredDerivations();
if (mode == Mode.full) {
// Compute gradient with respect to the predicted derivations
ensureExecuted();
if (computeExpectedCounts) {
expectedCounts = new HashMap<>();
ParserState.computeExpectedCounts(predDerivations, expectedCounts);
}
}
}
// Create all the derivations for the span [start, end).
@Override
protected void build(int start, int end) {
applyNonCatUnaryRules(start, end, start, parser.trie, new ArrayList<Derivation>(), new IntRef(0));
Set<String> cellsPruned = new HashSet<String>();
Set<String> cellsPruned = new HashSet<>();
applyCatUnaryRules(start, end, cellsPruned);
for (Map.Entry<String, List<Derivation>> entry : getChart()[start][end].entrySet())
for (Map.Entry<String, List<Derivation>> entry : chart[start][end].entrySet())
pruneCell(cellsPruned, entry.getKey(), start, end, entry.getValue());
}
private void pruneCell(Set<String> cellsPruned, String cat, int start, int end, List<Derivation> derivations) {
String cell = cellString(cat, start, end);
if (cellsPruned.contains(cell)) return;
cellsPruned.add(cell);
// Keep stats
if (derivations.size() > maxCellSize) {
maxCellSize = derivations.size();
maxCellDescription = String.format("[%s %s]", cat, getExample().spanString(start, end));
if (maxCellSize > 5000)
LogInfo.logs("BeamParser.pruneCell %s: %s entries", maxCellDescription, maxCellSize);
}
// The extra code blocks in here that set |deriv.maxXBeamPosition|
// are there to track, over the course of parsing, the lowest
// position at which any of a derivation's constituents ever
// placed on any of the relevant beams.
// Max beam position (before sorting)
for (int i = 0; i < derivations.size(); i++) {
Derivation deriv = derivations.get(i);
deriv.maxUnsortedBeamPosition = i;
if (deriv.children != null) {
for (Derivation child : deriv.children)
deriv.maxUnsortedBeamPosition = Math.max(deriv.maxUnsortedBeamPosition, child.maxUnsortedBeamPosition);
}
if (deriv.preSortBeamPosition == -1) {
// Need to be careful to only do this once since |pruneCell()|
// might be called several times for the same beam and the
// second time around we have already sorted once.
deriv.preSortBeamPosition = i;
}
}
Derivation.sortByScore(derivations);
// Max beam position (after sorting)
for (int i = 0; i < derivations.size(); i++) {
Derivation deriv = derivations.get(i);
deriv.maxBeamPosition = i;
if (deriv.children != null) {
for (Derivation child : deriv.children)
deriv.maxBeamPosition = Math.max(deriv.maxBeamPosition, child.maxBeamPosition);
}
deriv.postSortBeamPosition = i;
}
// Keep only the top hypotheses
int beamSize = getBeamSize();
while (derivations.size() > beamSize) {
derivations.remove(derivations.size() - 1);
fallOffBeam = true;
}
// Reduce memory
if (derivations instanceof ArrayList)
((ArrayList) derivations).trimToSize();
}
static String cellString(String cat, int start, int end) {
private static String cellString(String cat, int start, int end) {
return cat + ":" + start + ":" + end;
}
// Return number of new derivations added
private int applyRule(int start, int end, Rule rule, List<Derivation> children) {
if (Parser.opts.verbose >= 5)
LogInfo.logs("applyRule %s %s %s %s", start, end, rule, children);
if (Parser.opts.verbose >= 5) LogInfo.logs("applyRule %s %s %s %s", start, end, rule, children);
try {
if (getMode() == Mode.full) {
if (mode == Mode.full) {
StopWatchSet.begin(rule.getSemRepn());
List<Derivation> results = rule.sem.call(
getExample(),
DerivationStream results = rule.sem.call(ex,
new SemanticFn.CallInfo(rule.lhs, start, end, rule, ImmutableList.copyOf(children)));
StopWatchSet.end();
for (Derivation newDeriv : results) {
while (results.hasNext()) {
Derivation newDeriv = results.next();
featurizeAndScoreDerivation(newDeriv);
addToChart(newDeriv);
}
return results.size();
} else if (getMode() == Mode.bool) {
return results.estimatedSize();
} else if (mode == Mode.bool) {
Derivation deriv = new Derivation.Builder()
.cat(rule.lhs).start(start).end(end).rule(rule)
.children(ImmutableList.copyOf(children))
.formula(Formula.nullFormula)
.createDerivation();
.cat(rule.lhs).start(start).end(end).rule(rule)
.children(ImmutableList.copyOf(children))
.formula(Formula.nullFormula)
.createDerivation();
addToChart(deriv);
return 1;
} else {
@ -182,6 +166,14 @@ class BeamParserState extends ParserState {
}
}
// Don't prune the same cell more than once.
protected void pruneCell(Set<String> cellsPruned, String cat, int start, int end, List<Derivation> derivations) {
String cell = cellString(cat, start, end);
if (cellsPruned.contains(cell)) return;
cellsPruned.add(cell);
pruneCell(cell, derivations);
}
// Apply all unary rules with RHS category.
// Before applying each unary rule (rule.lhs -> rhsCat), we can prune the cell of rhsCat
// because we assume acyclicity, so rhsCat's cell will never grow.
@ -190,7 +182,7 @@ class BeamParserState extends ParserState {
if (!coarseAllows(rule.lhs, start, end))
continue;
String rhsCat = rule.rhs.get(0);
List<Derivation> derivations = getChart()[start][end].get(rhsCat);
List<Derivation> derivations = chart[start][end].get(rhsCat);
if (Parser.opts.verbose >= 5)
LogInfo.logs("applyCatUnaryRules %s %s %s %s", start, end, rule, derivations);
if (derivations == null) continue;
@ -228,7 +220,7 @@ class BeamParserState extends ParserState {
// apply the rule on all the children gathered during the walk.
if (i == end) {
for (Rule rule : node.rules) {
if(coarseAllows(rule.lhs, start, end)) {
if (coarseAllows(rule.lhs, start, end)) {
numNew.value += applyRule(start, end, rule, children);
}
}
@ -238,23 +230,86 @@ class BeamParserState extends ParserState {
// Advance terminal token
applyNonCatUnaryRules(
start, end, i + 1,
node.next(getExample().token(i)),
node.next(ex.token(i)),
children,
numNew);
// Advance non-terminal category
for (int j = i + 1; j <= end; j++) {
for (Map.Entry<String, List<Derivation>> entry : getChart()[i][j].entrySet()) {
for (Map.Entry<String, List<Derivation>> entry : chart[i][j].entrySet()) {
Trie nextNode = node.next(entry.getKey());
for (Derivation arg : entry.getValue()) {
children.add(arg);
applyNonCatUnaryRules(start, end, j, nextNode, children, numNew);
children.remove(children.size() - 1);
if (getMode() != Mode.full) break; // Only need one hypothesis
if (mode != Mode.full) break; // Only need one hypothesis
if (numNew.value >= BeamParser.opts.maxNewTreesPerSpan) return;
}
}
}
}
}
// -- Coarse state pruning --
// Remove any (cat, start, end) which isn't reachable from the
// (Rule.rootCat, 0, numTokens)
public void keepTopDownReachable() {
if (numTokens == 0) return;
Set<String> reachable = new HashSet<>();
collectReachable(reachable, Rule.rootCat, 0, numTokens);
// Remove all derivations associated with (cat, start, end) that aren't reachable.
for (int start = 0; start < numTokens; start++) {
for (int end = start + 1; end <= numTokens; end++) {
List<String> toRemoveCats = new LinkedList<>();
for (String cat : chart[start][end].keySet()) {
String key = catStartEndKey(cat, start, end);
if (!reachable.contains(key)) {
toRemoveCats.add(cat);
}
}
Collections.sort(toRemoveCats);
for (String cat : toRemoveCats) {
if (parser.verbose(4)) {
LogInfo.logs("Pruning chart %s(%s,%s)", cat, start, end);
}
chart[start][end].remove(cat);
}
}
}
}
private void collectReachable(Set<String> reachable, String cat, int start, int end) {
String key = catStartEndKey(cat, start, end);
if (reachable.contains(key)) return;
if (!chart[start][end].containsKey(cat)) {
// This should only happen for the root when there are no parses.
return;
}
reachable.add(key);
for (Derivation deriv : chart[start][end].get(cat)) {
for (Derivation subderiv : deriv.children) {
collectReachable(reachable, subderiv.cat, subderiv.start, subderiv.end);
}
}
}
private String catStartEndKey(String cat, int start, int end) {
return cat + ":" + start + ":" + end;
}
// For pruning with the coarse state
protected boolean coarseAllows(Trie node, int start, int end) {
if (coarseState == null) return true;
return SetUtils.intersects(
node.cats,
coarseState.chart[start][end].keySet());
}
protected boolean coarseAllows(String cat, int start, int end) {
if (coarseState == null) return true;
return coarseState.chart[start][end].containsKey(cat);
}
}

9
src/edu/stanford/nlp/sempre/BooleanValue.java
View File

@ -15,14 +15,15 @@ public class BooleanValue extends Value {
public LispTree toLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild("boolean");
tree.addChild(value+"");
tree.addChild(value + "");
return tree;
}
@Override public int hashCode() { return Boolean.valueOf(value).hashCode(); }
@Override public boolean equals(Object thatObj) {
if (!(thatObj instanceof BooleanValue)) return false;
BooleanValue that = (BooleanValue)thatObj;
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BooleanValue that = (BooleanValue) o;
return this.value == that.value;
}
}

77
src/edu/stanford/nlp/sempre/BoundedPriorityQueue.java Normal file
View File

@ -0,0 +1,77 @@
package edu.stanford.nlp.sempre;
import java.util.*;
/**
* Created by joberant on 3/27/14.
* A priority queue that holds no more than N elements
*/
public class BoundedPriorityQueue<E> extends TreeSet<E> {
private static final long serialVersionUID = 5724671156522771658L;
private int elementsLeft;
public BoundedPriorityQueue(int maxSize, Comparator<E> comparator) {
super(comparator);
this.elementsLeft = maxSize;
}
/**
* @return true if element was added, false otherwise
* */
@Override
public boolean add(E e) {
if (elementsLeft == 0 && size() == 0) {
// max size was initiated to zero => just return false
return false;
} else if (elementsLeft > 0) {
// queue isn't full => add element and decrement elementsLeft
boolean added = super.add(e);
if (added) {
elementsLeft--;
}
return added;
} else {
// there is already 1 or more elements => compare to the least
int compared = super.comparator().compare(e, this.last());
if (compared == -1) {
// new element is larger than the least in queue => pull the least and add new one to queue
pollLast();
super.add(e);
return true;
} else {
// new element is less than the least in queue => return false
return false;
}
}
}
public List<E> toList() {
List<E> res = new ArrayList<>();
for (E e : this)
res.add(e);
return res;
}
public static void main(String[] args) {
BoundedPriorityQueue<Integer> queue =
new BoundedPriorityQueue<>(5,
new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o1.compareTo(o2);
}
});
queue.add(10);
queue.add(8);
queue.add(4);
queue.add(12);
queue.add(3);
queue.add(7);
queue.add(9);
for (Integer num : queue) {
System.out.println(num);
}
}
}

409
src/edu/stanford/nlp/sempre/BridgeFn.java
View File

@ -1,409 +0,0 @@
package edu.stanford.nlp.sempre;
import edu.stanford.nlp.sempre.FbFormulasInfo.BinaryFormulaInfo;
import edu.stanford.nlp.sempre.MergeFormula.Mode;
import fig.basic.LispTree;
import fig.basic.LogInfo;
import fig.basic.Option;
import fig.basic.IOUtils;
import java.io.IOException;
import java.util.*;
/**
* Bridge between two derivations by type-raising one of them.
*
* @author jonathanberant
*/
public class BridgeFn extends SemanticFn {
private static final Formula intFormula = Formulas.fromLispTree(LispTree.proto.parseFromString("(fb:type.object.type fb:type.int)"));
private static final Formula floatFormula = Formulas.fromLispTree(LispTree.proto.parseFromString("(fb:type.object.type fb:type.float)"));
public static class Options {
@Option(gloss = "Verbose") public int verbose = 0;
@Option(gloss = "Whether to allow entity bridging with no binary string match") public boolean looseEntBridge = false;
@Option(gloss = "List of binary formulas to use during bridging") public String binariesFile = "";
}
public static Options opts = new Options();
private FbFormulasInfo fbFormulaInfo = null;
private String description;
private boolean headFirst;
private TextToTextMatcher textToTextMatcher;
private static HashSet<Formula> binariesToUse;
public void init(LispTree tree) {
super.init(tree);
if (tree.children.size() != 3)
throw new RuntimeException("Number of children is: " + tree.children.size());
if (!tree.child(2).value.equals("headFirst") && !tree.child(2).value.equals("headLast"))
throw new RuntimeException("Bad argument for head position: " + tree.child(2).value);
if (!tree.child(1).value.equals("unary") && !tree.child(1).value.equals("inject") && !tree.child(1).value.equals("entity"))
throw new RuntimeException("Bad description: " + tree.child(1).value);
this.description = tree.child(1).value;
headFirst = tree.child(2).value.equals("headFirst");
}
public BridgeFn() {
fbFormulaInfo = FbFormulasInfo.getSingleton();
textToTextMatcher = TextToTextMatcher.getSingleton();
binariesToUse = new HashSet<Formula>();
if (!opts.binariesFile.equals("")) {
readBinaries();
}
}
public static void readBinaries() {
for (String line : IOUtils.readLinesHard(opts.binariesFile)) {
if (line.startsWith("#")) continue;
if (line.equals("")) continue;
binariesToUse.add(Formula.fromString(line));
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BridgeFn bridgeFn = (BridgeFn) o;
if (headFirst != bridgeFn.headFirst) return false;
if (!description.equals(bridgeFn.description)) return false;
return true;
}
@Override
public List<Derivation> call(Example ex, Callable c) {
try {
if (description.equals("unary")) {
return bridgeUnary(ex, c);
} else if (description.equals("inject")) {
return injectIntoCvt(ex, c);
} else if (description.equals("entity")) {
return bridgeEntity(ex, c);
} else {
throw new RuntimeException("Invalid (expected unary, inject, or entity): " + description);
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
private boolean isCvt(Derivation headDeriv) {
if (!(headDeriv.formula instanceof JoinFormula))
return false;
JoinFormula join = (JoinFormula) headDeriv.formula;
if (join.relation instanceof LambdaFormula)
return true;
if (join.child instanceof JoinFormula || join.child instanceof MergeFormula)
return true;
return false;
}
// Return all the entity supertypes of |type|.
// TODO: make this more efficient.
private Set<String> getSupertypes(SemType type, Set<String> supertypes) {
if (type instanceof EntitySemType)
supertypes.addAll(fbFormulaInfo.getIncludedTypesInclusive(((EntitySemType) type).name));
else if (type instanceof UnionSemType)
for (SemType baseType : ((UnionSemType) type).baseTypes)
getSupertypes(baseType, supertypes);
else {
// FIXME HACK for when passing binary into lambda formula and
// getSuperTypes doesn't work
getSupertypes(SemType.fromString("topic"), supertypes);
//throw new RuntimeException("Unexpected type (must be unary): " + type);
}
return supertypes;
}
private List<Derivation> bridgeUnary(Example ex, Callable c) throws IOException {
List<Derivation> res = new ArrayList<Derivation>();
// Example: modifier[Hanks] head[movies]
Derivation headDeriv = headFirst ? c.child(0) : c.child(1);
Derivation modifierDeriv = !headFirst ? c.child(0) : c.child(1);
Set<String> headTypes = getSupertypes(headDeriv.type, new HashSet<String>());
Set<String> modifierTypes = getSupertypes(modifierDeriv.type, new HashSet<String>());
HashSet<String> usedBinaries = new HashSet<String>();
for (String modifierType : modifierTypes) { // For each head type...
Set<Formula> binaries;
if (binariesToUse.size() == 0)
binaries = fbFormulaInfo.getBinariesForType2(modifierType);
else
binaries = binariesToUse;
for (Formula binary : binaries) { // For each possible binary...
if (usedBinaries.contains(binary.toString()))
continue;
BinaryFormulaInfo binaryInfo = fbFormulaInfo.getBinaryInfo(binary);
if (opts.verbose >= 3)
LogInfo.logs("%s => %s", modifierType, binary);
if (headTypes.contains(binaryInfo.expectedType1)) {
Formula bridgedFormula = buildBridge(headDeriv.formula, modifierDeriv.formula, binary);
FbFormulasInfo.touchBinaryFormula(binary);
usedBinaries.add(binary.toString());
Derivation newDeriv = new Derivation.Builder()
.withCallable(c)
.formula(bridgedFormula)
.type(headDeriv.type)
.createDerivation();
if (SemanticFn.opts.trackLocalChoices) {
newDeriv.localChoices.add(
String.format(
"BridgeFn: %s %s --> %s %s --> %s %s",
headDeriv.startEndString(ex.getTokens()), headDeriv.formula,
ex.getTokens().subList(c.child(0).end, c.child(1).start),
binary,
modifierDeriv.startEndString(ex.getTokens()), modifierDeriv.formula));
}
// Add features
if (ex != null) {
newDeriv.addFeature("BridgeFn", "unary");
// Popularity of the binary
newDeriv.addFeatureWithBias("BridgeFn", "popularity", Math.log(binaryInfo.popularity + 1));
// head modifier POS tags
String headModifierOrder = headFirst ? "head-modifier" : "modifier-head";
newDeriv.addFeature("BridgeFn",
"order=" + headModifierOrder + "," +
"pos=" +
ex.languageInfo.getCanonicalPos(headDeriv.start) + "-" +
ex.languageInfo.getCanonicalPos(modifierDeriv.start));
addBinaryMatchFeatures(ex, modifierDeriv, binary, newDeriv); //HACKY
List<List<String>> exampleInfo = generateExampleInfo(ex, c); //this is not done in text to text matcher so done here
FeatureVector vector = textToTextMatcher.extractFeatures(
exampleInfo.get(0), exampleInfo.get(1), exampleInfo.get(2),
new HashSet<String>(binaryInfo.descriptions));
newDeriv.addFeatures(vector);
}
res.add(newDeriv);
}
}
}
return res;
}
//see whether the binary bridge can also be retrieved with the alignment lexicon
private void addBinaryMatchFeatures(Example ex, Derivation modifierDeriv, Formula binary, Derivation newDeriv) throws IOException {
if (!FeatureExtractor.containsDomain("bridgeBinaryMatch")) return;
for (int i = 0; i < ex.languageInfo.lemmaTokens.size(); ++i) {
if (i >= modifierDeriv.start && i < modifierDeriv.end)
continue;
String pos = ex.languageInfo.posTags.get(i);
String lemma = ex.languageInfo.lemmaTokens.get(i);
if (pos.startsWith("NN") || (pos.startsWith("VB") && !pos.equals("VBD-AUX")) || pos.equals("JJ") || pos.equals("IN")) {
LexiconFn fn = new LexiconFn();
fn.init(LispTree.proto.parseFromString("(LexiconFn binary)"));
Derivation child = new Derivation.Builder()
.cat("$CompositeRel").start(i).end(i + 1)
.children(new ArrayList<Derivation>())
.withStringFormulaFrom(lemma)
.createDerivation();
CallInfo c = new CallInfo("$Binary", i, i + 1, null, Collections.singletonList(child));
List<Derivation> derivations = fn.call(ex, c);
for (Derivation deriv : derivations) {
if (deriv.formula.equals(binary) || deriv.formula.equals(FbFormulasInfo.reverseFormula(binary))) {
if (opts.verbose >= 1) {
LogInfo.logs("BridgeFn: lemma %s matched bridged binary %s", lemma, binary);
}
newDeriv.addFeatures(deriv);
return;
}
}
}
}
}
//bridge without a unary - simply by looking at binaries leading to the entity and string matching binary description to example tokens/lemmas/stems
private List<Derivation> bridgeEntity(Example ex, Callable c) throws IOException {
List<Derivation> res = new ArrayList<Derivation>();
Derivation modifierDeriv = c.child(0);
Set<String> modifierTypes = getSupertypes(modifierDeriv.type, new HashSet<String>());
for (String modifierType : modifierTypes) { // For each head type...
Set<Formula> binaries = fbFormulaInfo.getBinariesForType2(modifierType);
for (Formula binary : binaries) { // For each possible binary...
BinaryFormulaInfo binaryInfo = fbFormulaInfo.getBinaryInfo(binary);
if (opts.verbose >= 3)
LogInfo.logs("%s => %s", modifierType, binary);
List<List<String>> exampleInfo = generateExampleInfo(ex, c); //this is not done in text to text matcher so done here
if (textToTextMatcher.existsTokenMatch(exampleInfo.get(0), exampleInfo.get(2), new HashSet<String>(binaryInfo.descriptions))
|| opts.looseEntBridge) {
Formula join = new JoinFormula(binary, modifierDeriv.formula);
FbFormulasInfo.touchBinaryFormula(binary);
Derivation newDeriv = new Derivation.Builder()
.withCallable(c)
.formula(join)
.type(new EntitySemType(binaryInfo.expectedType1))
.createDerivation();
if (opts.verbose >= 2)
LogInfo.logs("BridgeStringFn: %s", join);
//features
if (ex != null) {
newDeriv.addFeature("BridgeFn", "entity");
newDeriv.addFeatureWithBias("BridgeFn", "popularity", Math.log(binaryInfo.popularity + 1));
addBinaryMatchFeatures(ex, modifierDeriv, binary, newDeriv); //HACKY
FeatureVector textMatchFeatures = textToTextMatcher.extractFeatures(
exampleInfo.get(0), exampleInfo.get(1), exampleInfo.get(2),
new HashSet<String>(binaryInfo.descriptions));
newDeriv.addFeatures(textMatchFeatures);
//addTokenMatchFeatures(tokenStemFeatures.first(), newDeriv, "binary_token");
//addTokenMatchFeatures(tokenStemFeatures.second(), newDeriv, "binary_stem");
//addWordSimilarityFeatures(ex, newDeriv, binaryInfo); // (1) edit distance (2) word similarity
}
//newDeriv.localFeatureVector.add("bridge_lex_"+binaryInfo.expectedType1+"-->"+binary); //causes overfitting with 300 training examples
res.add(newDeriv);
}
}
}
return res;
}
//generate from example array of content word tokens/lemmas/stems that are not dominated by child derivations
private List<List<String>> generateExampleInfo(Example ex, Callable c) {
List<String> tokens = new ArrayList<String>();
List<String> posTags = new ArrayList<String>();
List<String> lemmas = new ArrayList<String>();
List<List<String>> res = new ArrayList<List<String>>();
res.add(tokens);
res.add(posTags);
res.add(lemmas);
Derivation modifierDeriv = headFirst ? c.child(1) : c.child(0);
for (int i = 0; i < ex.languageInfo.tokens.size(); ++i) {
if (i >= modifierDeriv.start && i < modifierDeriv.end) { //do not consider the modifier words {
continue;
}
tokens.add(ex.languageInfo.tokens.get(i));
posTags.add(ex.languageInfo.posTags.get(i));
lemmas.add(ex.languageInfo.lemmaTokens.get(i));
}
return res;
}
private List<Derivation> injectIntoCvt(Example ex, Callable c) {
List<Derivation> res = new ArrayList<Derivation>();
// Example: modifier[Braveheart] head[Mel Gibson plays in]
Derivation headDeriv = headFirst ? c.child(0) : c.child(1);
if (!isCvt(headDeriv)) //only works on cvts
return res;
Derivation modifierDeriv = !headFirst ? c.child(0) : c.child(1);
JoinFormula headFormula = (JoinFormula)Formulas.betaReduction(headDeriv.formula);
//find the type of the cvt node
Set<String> headTypes = Collections.singleton(fbFormulaInfo.getBinaryInfo(headFormula.relation).expectedType2);
Set<String> modifierTypes = getSupertypes(modifierDeriv.type, new HashSet<String>());
for (String modifierType : modifierTypes) {
Set<Formula> binaries = fbFormulaInfo.getAtomicBinariesForType2(modifierType); //here we use atomic binaries since we inject into a CVT
for (Formula binary : binaries) { // For each possible binary...
BinaryFormulaInfo info = fbFormulaInfo.getBinaryInfo(binary);
if (headTypes.contains(info.expectedType1)) {
Formula bridgedFormula = buildBridgeFromCvt(headFormula, modifierDeriv.formula, binary);
FbFormulasInfo.touchBinaryFormula(binary);
Derivation newDeriv = new Derivation.Builder()
.withCallable(c)
.formula(bridgedFormula)
.type(headDeriv.type)
.createDerivation();
if (opts.verbose >= 3)
LogInfo.logs("BridgeFn: injecting %s to %s --> %s ", modifierDeriv.formula, headFormula, bridgedFormula);
if (ex != null) {
String headModifierOrder = headFirst ? "head-modifier" : "modifier-head";
newDeriv.addFeature("BridgeFn",
"inject_order=" + headModifierOrder + "," + "pos=" +
ex.languageInfo.getCanonicalPos(headDeriv.start) + "-" +
ex.languageInfo.getCanonicalPos(modifierDeriv.start));
}
newDeriv.addFeature("BridgeFn", "binary=" + binary);
res.add(newDeriv);
}
}
}
return res;
}
// Checks whether "var" is used as a binary in "formula"
private boolean varIsBinary(Formula formula, String var) {
boolean isBinary = false;
LispTree tree = formula.toLispTree();
VariableFormula vf = new VariableFormula(var);
for (LispTree child : tree.children) {
if (child.isLeaf())
continue;
if (child.children.size() == 2 && vf.equals(Formulas.fromLispTree(child.child(0)))) {
isBinary = true;
break;
}
if (varIsBinary(Formulas.fromLispTree(child), var)) {
isBinary = true;
break;
}
}
return isBinary;
}
private Formula buildBridge(Formula headFormula, Formula modifierFormula, Formula binary) {
// Handle cases like "what state has the most cities" where "has the" is mapped
// to "contains" predicate via bridging but "most" triggers a nested lambda w/
// argmax on a count relation
// (Corresponds to $MetaMetaOperator in grammar)
if (modifierFormula instanceof LambdaFormula) {
LambdaFormula lf = (LambdaFormula) modifierFormula;
if (varIsBinary(lf, lf.var)) {
Formula newBinary = Formulas.lambdaApply(lf, binary);
if (newBinary instanceof LambdaFormula) {
Formula result = Formulas.lambdaApply((LambdaFormula) newBinary, headFormula);
return result;
}
}
}
Formula join = new JoinFormula(binary, modifierFormula);
Formula merge = new MergeFormula(Mode.and, headFormula, join);
//Don't merge on ints and floats
return (headFormula.equals(intFormula) || headFormula.equals(floatFormula)) ? join : merge;
}
private Formula buildBridgeFromCvt(JoinFormula headFormula, Formula modifierFormula, Formula binary) {
Formula join = new JoinFormula(binary, modifierFormula);
Formula merge = new MergeFormula(Mode.and, headFormula.child, join);
return new JoinFormula(headFormula.relation, merge);
}
}

44
src/edu/stanford/nlp/sempre/Builder.java
View File

@ -1,6 +1,7 @@
package edu.stanford.nlp.sempre;
import com.google.common.base.Strings;
import fig.basic.Option;
import fig.basic.Utils;
@ -12,9 +13,9 @@ import fig.basic.Utils;
*/
public class Builder {
public static class Options {
@Option public String packageName = "edu.stanford.nlp.sempre";
@Option public String inParamsPath;
@Option public String executor = "SparqlExecutor";
@Option public String executor = "JavaExecutor";
@Option public String valueEvaluator = "ExactValueEvaluator";
@Option public String parser = "BeamParser";
}
@ -22,6 +23,7 @@ public class Builder {
public Grammar grammar;
public Executor executor;
public ValueEvaluator valueEvaluator;
public FeatureExtractor extractor;
public Parser parser;
public Params params;
@ -29,6 +31,7 @@ public class Builder {
public void build() {
grammar = null;
executor = null;
valueEvaluator = null;
extractor = null;
parser = null;
params = null;
@ -45,19 +48,19 @@ public class Builder {
// Executor
if (executor == null)
executor = (Executor) Utils.newInstanceHard(opts.packageName + "." + opts.executor);
executor = (Executor) Utils.newInstanceHard(SempreUtils.resolveClassName(opts.executor));
// Feature extractors
// Value evaluator
if (valueEvaluator == null)
valueEvaluator = (ValueEvaluator) Utils.newInstanceHard(SempreUtils.resolveClassName(opts.valueEvaluator));
// Feature extractor
if (extractor == null)
extractor = new FeatureExtractor(executor);
// Parser
if (parser == null) {
if(opts.parser.equals("BeamParser"))
parser = new BeamParser(grammar, extractor, executor);
else
throw new RuntimeException("Illegal parser: " + opts.parser);
}
if (parser == null)
parser = buildParser(new Parser.Spec(grammar, extractor, executor, valueEvaluator));
// Parameters
if (params == null) {
@ -66,4 +69,25 @@ public class Builder {
params.read(opts.inParamsPath);
}
}
public static Parser buildParser(Parser.Spec spec) {
switch (opts.parser) {
case "BeamParser":
return new BeamParser(spec);
case "ReinforcementParser":
return new ReinforcementParser(spec);
case "FloatingParser":
return new FloatingParser(spec);
default:
// Try instantiating by name
try {
Class<?> parserClass = Class.forName(SempreUtils.resolveClassName(opts.parser));
return (Parser) parserClass.getConstructor(spec.getClass()).newInstance(spec);
} catch (ClassNotFoundException e1) {
throw new RuntimeException("Illegal parser: " + opts.parser);
} catch (Exception e) {
throw new RuntimeException("Error while instantiating parser: " + opts.parser + "\n" + e);
}
}
}
}

14
src/edu/stanford/nlp/sempre/CallFormula.java
View File

@ -8,6 +8,7 @@ import java.util.List;
/**
* A CallFormula represents a function call.
* See JavaExecutor for the semantics of this formula.
* (call func arg_1 ... arg_k)
*
* @author Percy Liang
@ -44,6 +45,17 @@ public class CallFormula extends Formula {
return new CallFormula(newFunc, newArgs);
}
@Override
public List<Formula> mapToList(Function<Formula, List<Formula>> transform, boolean alwaysRecurse) {
List<Formula> res = transform.apply(this);
if (res.isEmpty() || alwaysRecurse) {
res.addAll(func.mapToList(transform, alwaysRecurse));
for (Formula arg : args)
res.addAll(arg.mapToList(transform, alwaysRecurse));
}
return res;
}
@Override
public boolean equals(Object thatObj) {
if (!(thatObj instanceof CallFormula)) return false;
@ -52,7 +64,7 @@ public class CallFormula extends Formula {
if (!this.args.equals(that.args)) return false;
return true;
}
public int computeHashCode() {
int hash = 0x7ed55d16;
hash = hash * 0xd3a2646c + func.hashCode();

16
src/edu/stanford/nlp/sempre/CallTypeInfo.java Normal file
View File

@ -0,0 +1,16 @@
package edu.stanford.nlp.sempre;
import java.util.List;
// Type information for each function in CallFormula.
public class CallTypeInfo {
public final String func;
public final List<SemType> argTypes;
public final SemType retType;
public CallTypeInfo(String func, List<SemType> argTypes, SemType retType) {
this.func = func;
this.argTypes = argTypes;
this.retType = retType;
}
}

42
src/edu/stanford/nlp/sempre/CanonicalNames.java Normal file
View File

@ -0,0 +1,42 @@
package edu.stanford.nlp.sempre;
import java.util.*;
/**
* List of canonical names that we borrowed from Freebase.
*
* These names and helper methods are independent from the Freebase schema.
*
* @author ppasupat
*/
public final class CanonicalNames {
private CanonicalNames() { }
// Standard type names
public static final String BOOLEAN = "fb:type.boolean";
public static final String INT = "fb:type.int";
public static final String FLOAT = "fb:type.float";
public static final String DATE = "fb:type.datetime";
public static final String TEXT = "fb:type.text";
public static final String NUMBER = "fb:type.number";
public static final String ENTITY = "fb:common.topic";
public static final String ANY = "fb:type.any";
public static final List<String> PRIMITIVES = Collections.unmodifiableList(
Arrays.asList(BOOLEAN, INT, FLOAT, DATE, TEXT, NUMBER));
// Standard relations
public static final String TYPE = "fb:type.object.type";
public static final String NAME = "fb:type.object.name";
// Return whether |property| is the name of a reverse property.
// Convention: ! is the prefix for reverses.
public static boolean isReverseProperty(String property) {
return property.startsWith("!") && !property.equals("!=");
}
public static String reverseProperty(String property) {
if (isReverseProperty(property)) return property.substring(1);
else return "!" + property;
}
}

122
src/edu/stanford/nlp/sempre/ChartParserState.java Normal file
View File

@ -0,0 +1,122 @@
package edu.stanford.nlp.sempre;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import fig.basic.LogInfo;
import fig.basic.MapUtils;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Actually does the parsing. Main method is infer(), whose job is to fill in
*
* @author Roy Frostig
* @author Percy Liang
*/
public abstract class ChartParserState extends ParserState {
// cell (start, end, category) -> list of derivations (sorted by decreasing score) [beam]
protected final Map<String, List<Derivation>>[][] chart;
// For visualizing how chart is filled
List<CatSpan> chartFillingList = new ArrayList<>();
protected String[][] phrases; // the phrases in the example
@SuppressWarnings({ "unchecked" })
public ChartParserState(Parser parser, Params params, Example ex, boolean computeExpectedCounts) {
super(parser, params, ex, computeExpectedCounts);
// Initialize the chart.
this.chart = (HashMap<String, List<Derivation>>[][])
Array.newInstance(HashMap.class, numTokens, numTokens + 1);
this.phrases = new String[numTokens][numTokens + 1];
for (int start = 0; start < numTokens; start++) {
StringBuilder sb = new StringBuilder();
for (int end = start + 1; end <= numTokens; end++) {
if (end - start > 1)
sb.append(' ');
sb.append(this.ex.languageInfo.tokens.get(end - 1));
phrases[start][end] = sb.toString();
chart[start][end] = new HashMap<>();
}
}
}
public void clearChart() {
for (int start = 0; start < numTokens; start++) {
for (int end = start + 1; end <= numTokens; end++) {
chart[start][end].clear();
}
}
}
// Call this method in infer()
protected void setPredDerivations() {
predDerivations.clear();
predDerivations.addAll(MapUtils.get(chart[0][numTokens], Rule.rootCat, Derivation.emptyList));
}
private void visualizeChart() {
for (int len = 1; len <= numTokens; ++len) {
for (int i = 0; i + len <= numTokens; ++i) {
for (String cat : chart[i][i + len].keySet()) {
List<Derivation> derivations = chart[i][i + len].get(cat);
for (Derivation deriv : derivations) {
LogInfo.logs("ParserState.visualize: %s(%s:%s): %s", cat, i, i + len, deriv);
}
}
}
}
}
void addToChart(Derivation deriv) {
if (parser.verbose(3)) LogInfo.logs("addToChart %s: %s", deriv.cat, deriv);
List<Derivation> derivations = chart[deriv.start][deriv.end].get(deriv.cat);
if (chart[deriv.start][deriv.end].get(deriv.cat) == null)
chart[deriv.start][deriv.end].put(deriv.cat, derivations = new ArrayList<>());
derivations.add(deriv);
totalGeneratedDerivs++;
if (Parser.opts.visualizeChartFilling) {
chartFillingList.add(new CatSpan(deriv.start, deriv.end, deriv.cat));
}
}
// TODO(joberant): move to visualization utility class
public static class CatSpan {
@JsonProperty
public final int start;
@JsonProperty public final int end;
@JsonProperty public final String cat;
@JsonCreator
public CatSpan(@JsonProperty("start") int start, @JsonProperty("end") int end,
@JsonProperty("cat") String cat) {
this.start = start;
this.end = end;
this.cat = cat;
}
}
public static class ChartFillingData {
@JsonProperty public final String id;
@JsonProperty public final String utterance;
@JsonProperty public final int numOfTokens;
@JsonProperty public final List<CatSpan> catSpans;
@JsonCreator
public ChartFillingData(@JsonProperty("id") String id, @JsonProperty("catspans") List<CatSpan> catSpans,
@JsonProperty("utterance") String utterance, @JsonProperty("numOfTokens") int numOfTokens) {
this.id = id;
this.utterance = utterance;
this.numOfTokens = numOfTokens;
this.catSpans = catSpans;
}
}
}

284
src/edu/stanford/nlp/sempre/CoarseParser.java Normal file
View File

@ -0,0 +1,284 @@
package edu.stanford.nlp.sempre;
import com.google.common.base.Joiner;
import fig.basic.LogInfo;
import fig.basic.MapUtils;
import fig.basic.Pair;
import fig.basic.StopWatch;
import java.lang.reflect.Array;
import java.util.*;
/**
* Parser that only has information on what categories can parse what spans
* Does not hold backpointers for getting full parse, only reachability information
* Important: assumes that the grammar is binary
* Independent from the Parser code and therefore there is duplicate code (traverse(), keepTopDownReachable())
* @author jonathanberant
*/
public class CoarseParser {
public final Grammar grammar;
private Map<Pair<String, String>, Set<String>> rhsToLhsMap;
ArrayList<Rule> catUnaryRules; // Unary rules with category on RHS
Map<String, List<Rule>> terminalsToRulesList = new HashMap<>();
public CoarseParser(Grammar grammar) {
this.grammar = grammar;
catUnaryRules = new ArrayList<>();
rhsToLhsMap = new HashMap<>();
Map<String, List<Rule>> graph = new HashMap<>(); // Node from LHS to list of rules
for (Rule rule : grammar.rules) {
if (rule.rhs.size() > 2)
throw new RuntimeException("We assume that the grammar is binarized, rule: " + rule);
if (rule.isCatUnary())
MapUtils.addToList(graph, rule.lhs, rule);
else if (rule.rhs.size() == 2) { // binary grammar
MapUtils.addToSet(rhsToLhsMap, Pair.newPair(rule.rhs.get(0), rule.rhs.get(1)), rule.lhs);
} else {
assert rule.isRhsTerminals();
MapUtils.addToList(terminalsToRulesList, Joiner.on(' ').join(rule.rhs), rule);
}
}
// Topologically sort catUnaryRules so that B->C occurs before A->B
Map<String, Boolean> done = new HashMap<>();
for (String node : graph.keySet())
traverse(catUnaryRules, node, graph, done);
LogInfo.logs("Coarse parser: %d catUnaryRules (sorted), %d nonCatUnaryRules", catUnaryRules.size(), grammar.rules.size() - catUnaryRules.size());
}
/** Helper function for transitive closure of unary rules. */
private void traverse(List<Rule> catUnaryRules,
String node,
Map<String, List<Rule>> graph,
Map<String, Boolean> done) {
Boolean d = done.get(node);
if (Boolean.TRUE.equals(d)) return;
if (Boolean.FALSE.equals(d))
throw new RuntimeException("Found cycle of unaries involving " + node);
done.put(node, false);
for (Rule rule : MapUtils.getList(graph, node)) {
traverse(catUnaryRules, rule.rhs.get(0), graph, done);
catUnaryRules.add(rule);
}
done.put(node, true);
}
public CoarseParserState getCoarsePrunedChart(Example ex) {
CoarseParserState res = new CoarseParserState(ex, this);
res.infer();
return res;
}
class CoarseParserState {
private Map<String, List<CategorySpan>>[][] chart;
public final Example example;
public final CoarseParser parser;
private int numTokens;
private long time;
private String[][] phrases;
@SuppressWarnings({ "unchecked" })
public CoarseParserState(Example example, CoarseParser parser) {
this.example = example;
this.parser = parser;
numTokens = example.numTokens();
// Initialize the chart.
this.chart = (HashMap<String, List<CategorySpan>>[][])
Array.newInstance(
HashMap.class,
numTokens, numTokens + 1);
phrases = new String[numTokens][numTokens + 1];
for (int start = 0; start < numTokens; start++) {
StringBuilder sb = new StringBuilder();
for (int end = start + 1; end <= numTokens; end++) {
if (end - start > 1)
sb.append(' ');
sb.append(example.languageInfo.tokens.get(end - 1));
phrases[start][end] = sb.toString();
chart[start][end] = new HashMap<>();
}
}
}
public long getCoarseParseTime() { return time; }
public void infer() {
StopWatch watch = new StopWatch();
watch.start();
// parse with rules with tokens or RHS
parseTokensAndPhrases();
// complete bottom up parsing
for (int len = 1; len <= numTokens; len++)
for (int i = 0; i + len <= numTokens; i++)
build(i, i + len);
// prune away things that are not reachable from the top
keepTopDownReachable();
watch.stop();
time = watch.getCurrTimeLong();
}
public boolean coarseAllows(String cat, int start, int end) {
return chart[start][end].containsKey(cat);
}
private void build(int start, int end) {
handleBinaryRules(start, end);
handleUnaryRules(start, end);
}
private void parseTokensAndPhrases() {
for (int i = 0; i < numTokens; ++i) {
addToChart(Rule.tokenCat, i, i + 1);
addToChart(Rule.lemmaTokenCat, i, i + 1);
}
for (int i = 0; i < numTokens; i++) {
for (int j = i + 1; j <= numTokens; j++) {
addToChart(Rule.phraseCat, i, j);
addToChart(Rule.lemmaPhraseCat, i, j);
}
}
}
private void addToChart(String cat, int start, int end) {
if (Parser.opts.verbose >= 5)
LogInfo.logs("Adding to chart %s(%s,%s)", cat, start, end);
MapUtils.putIfAbsent(chart[start][end], cat, new ArrayList<CategorySpan>());
}
private void addToChart(String parentCat, String childCat, int start, int end) {
if (Parser.opts.verbose >= 5)
LogInfo.logs("Adding to chart %s(%s,%s)-->%s(%s,%s)", parentCat, start, end, childCat, start, end);
MapUtils.addToList(chart[start][end], parentCat, new CategorySpan(childCat, start, end)); }
private void addToChart(String parentCat, String leftCat, String rightCat, int start, int i, int end) {
if (Parser.opts.verbose >= 5)
LogInfo.logs("Adding to chart %s(%s,%s)-->%s(%s,%s) %s(%s,%s)", parentCat, start, end, leftCat, start, i, rightCat, i, end);
MapUtils.addToList(chart[start][end], parentCat, new CategorySpan(leftCat, start, i));
MapUtils.addToList(chart[start][end], parentCat, new CategorySpan(rightCat, i, end));
}
private void handleBinaryRules(int start, int end) {
for (int i = start + 1; i < end; ++i) {
List<String> left = new ArrayList<>(chart[start][i].keySet());
List<String> right = new ArrayList<>(chart[i][end].keySet());
if (i - start == 1) left.add(phrases[start][i]); // handle single terminal
if (end - i == 1) right.add(phrases[i][end]); // handle single terminal
for (String l : left) {
for (String r : right) {
Set<String> parentCats = rhsToLhsMap.get(Pair.newPair(l, r));
if (parentCats != null) {
for (String parentCat : parentCats) {
addToChart(parentCat, l, r, start, i, end);
}
}
}
}
}
}
private void handleUnaryRules(int start, int end) {
// terminals on RHS
for (Rule rule : MapUtils.get(terminalsToRulesList, phrases[start][end], Collections.<Rule>emptyList())) {
addToChart(rule.lhs, start, end);
}
// catUnaryRules
for (Rule rule : parser.catUnaryRules) {
String rhsCat = rule.rhs.get(0);
if (chart[start][end].containsKey(rhsCat)) {
addToChart(rule.lhs, rhsCat, start, end);
}
}
}
public void keepTopDownReachable() {
if (numTokens == 0) return;
Set<CategorySpan> reachable = new HashSet<CategorySpan>();
collectReachable(reachable, new CategorySpan(Rule.rootCat, 0, numTokens));
// Remove all derivations associated with (cat, start, end) that aren't reachable.
for (int start = 0; start < numTokens; start++) {
for (int end = start + 1; end <= numTokens; end++) {
List<String> toRemoveCats = new LinkedList<String>();
for (String cat : chart[start][end].keySet()) {
if (!reachable.contains(new CategorySpan(cat, start, end))) {
toRemoveCats.add(cat);
}
}
Collections.sort(toRemoveCats);
for (String cat : toRemoveCats) {
if (Parser.opts.verbose >= 5)
LogInfo.logs("Pruning chart %s(%s,%s)", cat, start, end);
chart[start][end].remove(cat);
}
}
}
}
private void collectReachable(Set<CategorySpan> reachable, CategorySpan catSpan) {
if (reachable.contains(catSpan))
return;
if (!chart[catSpan.start][catSpan.end].containsKey(catSpan.cat)) {
// This should only happen for the root when there are no parses.
return;
}
reachable.add(catSpan);
for (CategorySpan childCatSpan : chart[catSpan.start][catSpan.end].get(catSpan.cat)) {
collectReachable(reachable, childCatSpan);
}
}
}
class CategorySpan {
public final String cat;
public final int start;
public final int end;
public CategorySpan(String cat, int start, int end) {
this.cat = cat;
this.start = start;
this.end = end;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((cat == null) ? 0 : cat.hashCode());
result = prime * result + end;
result = prime * result + start;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
CategorySpan other = (CategorySpan) obj;
if (cat == null) {
if (other.cat != null)
return false;
} else if (!cat.equals(other.cat))
return false;
if (end != other.end)
return false;
if (start != other.start)
return false;
return true;
}
}
}

50
src/edu/stanford/nlp/sempre/Colorizer.java Normal file
View File

@ -0,0 +1,50 @@
package edu.stanford.nlp.sempre;
/**
* Tools for colorizing output to console so easier to read
*
* @author Ziang Xie
*/
public class Colorizer {
public Colorizer() { }
public String colorize(String s, String color) {
String cp = "";
// NOTE JDK 7+ feature
switch (color) {
case "black":
cp = "\u001B[30m";
break;
case "red":
cp = "\u001B[31m";
break;
case "green":
cp = "\u001B[32m";
break;
case "yellow":
cp = "\u001B[33m";
break;
case "blue":
cp = "\u001B[34m";
break;
case "purple":
cp = "\u001B[35m";
break;
case "cyan":
cp = "\u001B[36m";
break;
case "white":
cp = "\u001B[37m";
break;
default:
throw new RuntimeException("Invalid color: " + color);
}
if (cp.equals(""))
return s;
return cp + s + "\u001B[0m";
}
}

45
src/edu/stanford/nlp/sempre/ConcatFn.java
View File

@ -1,10 +1,6 @@
package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.Collections;
import java.util.List;
/**
* Takes two strings and returns their concatenation.
*
@ -13,30 +9,31 @@ import java.util.List;
public class ConcatFn extends SemanticFn {
String delim;
public ConcatFn() { }
public ConcatFn(String delim) {
this.delim = delim;
}
public void init(LispTree tree) {
super.init(tree);
delim = tree.child(1).value;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ConcatFn concatFn = (ConcatFn) o;
if (!delim.equals(concatFn.delim)) return false;
return true;
}
public List<Derivation> call(Example ex, Callable c) {
StringBuilder out = new StringBuilder();
for (int i = 0; i < c.getChildren().size(); i++) {
if (i > 0) out.append(delim);
out.append(c.childStringValue(i));
}
return Collections.singletonList(
new Derivation.Builder()
.withCallable(c)
.withStringFormulaFrom(out.toString())
.createDerivation());
public DerivationStream call(Example ex, final Callable c) {
return new SingleDerivationStream() {
@Override
public Derivation createDerivation() {
StringBuilder out = new StringBuilder();
for (int i = 0; i < c.getChildren().size(); i++) {
if (i > 0) out.append(delim);
out.append(c.childStringValue(i));
}
return new Derivation.Builder()
.withCallable(c)
.withStringFormulaFrom(out.toString())
.createDerivation();
}
};
}
}

56
src/edu/stanford/nlp/sempre/ConstantFn.java
View File

@ -2,9 +2,6 @@ package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.Collections;
import java.util.List;
/**
* Just returns a fixed logical formula.
*
@ -26,43 +23,26 @@ public class ConstantFn extends SemanticFn {
if (2 < tree.children.size())
this.type = SemType.fromLispTree(tree.child(2));
else {
this.type = crudeInferType(formula);
this.type = TypeInference.inferType(formula);
}
if (!this.type.isValid())
throw new RuntimeException("ConstantFn: " + formula + " does not type check");
}
private SemType crudeInferType(Formula formula) {
// Try to infer the type
if (formula instanceof ValueFormula) {
Value value = ((ValueFormula)formula).value;
if (value instanceof NumberValue) return SemType.numberType;
else if (value instanceof StringValue) return SemType.stringType;
else if (value instanceof DateValue) return SemType.dateType;
else if (value instanceof NameValue) return SemType.entityType;
} else if (formula instanceof LambdaFormula) {
return new FuncSemType(SemType.topType, crudeInferType(((LambdaFormula)formula).body));
}
throw new RuntimeException("Can't infer type of " + formula);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ConstantFn that = (ConstantFn) o;
if (!formula.equals(that.formula)) return false;
if (!type.equals(that.type)) return false;
return true;
}
public List<Derivation> call(Example ex, Callable c) {
Derivation deriv = new Derivation.Builder()
.withCallable(c)
.formula(formula)
.type(type)
.createDerivation();
if (FeatureExtractor.containsDomain("constant"))
deriv.addFeature("constant", ex.phraseString(c.getStart(), c.getEnd()) + " --- " + formula.toString());
return Collections.singletonList(deriv);
public DerivationStream call(final Example ex, final Callable c) {
return new SingleDerivationStream() {
@Override
public Derivation createDerivation() {
Derivation res = new Derivation.Builder()
.withCallable(c)
.formula(formula)
.type(type)
.createDerivation();
// don't generate feature if it is not grounded to a string
if (FeatureExtractor.containsDomain("constant") && c.getStart() != -1)
res.addFeature("constant", ex.phraseString(c.getStart(), c.getEnd()) + " --- " + formula.toString());
return res;
}
};
}
}

123
src/edu/stanford/nlp/sempre/ContextFn.java Normal file
View File

@ -0,0 +1,123 @@
package edu.stanford.nlp.sempre;
import java.util.*;
import fig.basic.*;
/**
* Produces predicates (like LexiconFn) but do it from the logical forms
* in the context (inspects the ContextValue of the example).
*
* Takes depth, restrictType, and forbiddenTypes arguments allowing you
* to specify the depth/size and type of (formula) subtrees that you want to
* extract from the context.
*
* ONLY USE WITH TYPES!!
*
* E.g.,
*
* (rule $X (context) (ContextFn (depth 0) (type fb:type.any))
* would extract any unary/entity.
*
* (rule $X (context) (ContextFn (depth 1) (type (-> fb:type.any
* fb:type.any)) (forbidden (-> fb:type.any fb:type.something)) would
* extract all binaries except those with arg1 of type fb:type.something
*
* @author William Hamilton
*/
// TODO(Will): Reintegrate useful functionality from old implementation.
public class ContextFn extends SemanticFn {
// the depth/size of subtrees to extract
private int depth;
// the type that you want to extract
private SemType restrictType = SemType.topType;
// set of types to not extract (overrides restrictType).
// For example, if restrict type is very general (e.g., (-> type.any type.any))
// and you don't want some specific subtype (e.g., (-> type.something type.any))
// then you would say specify (forbidden (-> type.something type.any))
// and all subtypes of (-> type.any type.any) would be permissible
// except the forbidden one(s).
private Set<SemType> forbiddenTypes = new HashSet<SemType>();
public void init(LispTree tree) {
super.init(tree);
for (int i = 1; i < tree.children.size(); i++) {
LispTree arg = tree.child(i);
if ("type".equals(arg.child(0).value)) {
restrictType = SemType.fromLispTree(arg.child(1));
} else if ("depth".equals(arg.child(0).value)) {
depth = Integer.parseInt(arg.child(1).value);
} else if ("forbidden".equals(arg.child(0).value)) {
forbiddenTypes.add(SemType.fromLispTree(arg.child(1)));
} else {
throw new RuntimeException("Unknown argument: " + arg);
}
}
}
public DerivationStream call(final Example ex, final Callable c) {
return new MultipleDerivationStream() {
int index = 0;
List<Formula> formulas;
public Derivation createDerivation() {
if (ex.context == null) return null;
if (formulas == null) {
formulas = new ArrayList<Formula>();
for (int i = ex.context.exchanges.size() - 1; i >= 0; i--) {
ContextValue.Exchange e = ex.context.exchanges.get(i);
extractFormulas(e.formula.toLispTree());
}
}
if (index >= formulas.size()) return null;
Formula formula = formulas.get(index++);
for (SemType forbiddenType : forbiddenTypes) {
if (TypeInference.inferType(formula).meet(forbiddenType).isValid())
return null;
}
return new Derivation.Builder()
.withCallable(c)
.formula(formula)
.type(TypeInference.inferType(formula))
.createDerivation();
}
private void addFormula(Formula formula) {
if (formulas.contains(formula))
return;
formulas.add(formula);
}
// Extract from the logical form.
private void extractFormulas(LispTree formula) {
if (correctDepth(formula, 0) && typeCheck(formula)) {
addFormula(Formulas.fromLispTree(formula));
}
if (formula.isLeaf())
return;
for (LispTree child : formula.children)
extractFormulas(child);
}
private boolean correctDepth(LispTree formula, int currentLevel) {
if (formula.isLeaf()) {
return currentLevel == depth;
} else {
boolean isCorrect = true;
for (LispTree child : formula.children)
isCorrect = isCorrect && correctDepth(child, currentLevel + 1);
return isCorrect;
}
}
private boolean typeCheck(LispTree treeFormula) {
Formula formula = Formulas.fromLispTree(treeFormula);
SemType type = TypeInference.inferType(formula);
type = restrictType.meet(type);
return type.isValid();
}
};
}
}

144
src/edu/stanford/nlp/sempre/ContextValue.java Normal file
View File

@ -0,0 +1,144 @@
package edu.stanford.nlp.sempre;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
import fig.basic.LispTree;
import java.util.*;
/**
* Represents the discourse context (time, place, history of exchanges).
* This is part of an Example and used by ContextFn.
*
* @author Percy Liang
*/
public class ContextValue extends Value {
// A single exchange between the user and the system
// Note: we are not storing the entire derivation right now.
public static class Exchange {
public final String utterance;
public final Formula formula;
public final Value value;
public Exchange(String utterance, Formula formula, Value value) {
this.utterance = utterance;
this.formula = formula;
this.value = value;
}
public Exchange(LispTree tree) {
utterance = tree.child(1).value;
formula = Formulas.fromLispTree(tree.child(2));
value = Values.fromLispTree(tree.child(3));
}
public LispTree toLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild("exchange");
tree.addChild(utterance);
tree.addChild(formula.toLispTree());
tree.addChild(value.toLispTree());
return tree;
}
@Override public String toString() { return toLispTree().toString(); }
}
public final String user;
public final DateValue date;
public final List<Exchange> exchanges; // List of recent exchanges with the user
public final KnowledgeGraph graph; // Mini-knowledge graph that captures the context
public ContextValue withDate(DateValue newDate) {
return new ContextValue(user, newDate, exchanges, graph);
}
public ContextValue withNewExchange(List<Exchange> newExchanges) {
return new ContextValue(user, date, newExchanges, graph);
}
public ContextValue withGraph(KnowledgeGraph newGraph) {
return new ContextValue(user, date, exchanges, newGraph);
}
public ContextValue(String user, DateValue date, List<Exchange> exchanges, KnowledgeGraph graph) {
this.user = user;
this.date = date;
this.exchanges = exchanges;
this.graph = graph;
}
public ContextValue(String user, DateValue date, List<Exchange> exchanges) {
this(user, date, exchanges, null);
}
public ContextValue(KnowledgeGraph graph) {
this(null, null, null, graph);
}
// Example:
// (context (user pliang)
// (date 2014 4 20)
// (exchange "when was chopin born" (!fb:people.person.date_of_birth fb:en.frederic_chopin) (date 1810 2 22))
// (graph NaiveKnowledgeGraph ((string Obama) (string "born in") (string Hawaii)) ...))
public ContextValue(LispTree tree) {
String user = null;
DateValue date = null;
KnowledgeGraph graph = null;
exchanges = new ArrayList<Exchange>();
for (int i = 1; i < tree.children.size(); i++) {
String key = tree.child(i).child(0).value;
if (key.equals("user")) {
user = tree.child(i).child(1).value;
} else if (key.equals("date")) {
date = new DateValue(tree.child(i));
} else if (key.equals("graph")) {
graph = KnowledgeGraph.fromLispTree(tree.child(i));
} else if (key.equals("exchange")) {
exchanges.add(new Exchange(tree.child(i)));
} else {
throw new RuntimeException("Invalid: " + tree.child(i));
}
}
this.user = user;
this.date = date;
this.graph = graph;
}
public LispTree toLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild("context");
if (user != null)
tree.addChild(LispTree.proto.newList("user", user));
if (date != null)
tree.addChild(date.toLispTree());
if (graph != null)
tree.addChild(graph.toLispTree());
for (Exchange e : exchanges)
tree.addChild(LispTree.proto.newList("exchange", e.toLispTree()));
return tree;
}
@Override public int hashCode() {
int hash = 0x7ed55d16;
hash = hash * 0xd3a2646c + user.hashCode();
hash = hash * 0xd3a2646c + date.hashCode();
hash = hash * 0xd3a2646c + exchanges.hashCode();
hash = hash * 0xd3a2646c + graph.hashCode();
return hash;
}
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ContextValue that = (ContextValue) o;
if (!this.user.equals(that.user)) return false;
if (!this.date.equals(that.date)) return false;
if (!this.exchanges.equals(that.exchanges)) return false;
if (!this.graph.equals(that.graph)) return false;
return true;
}
@JsonValue
public String toString() { return toLispTree().toString(); }
@JsonCreator
public static ContextValue fromString(String str) {
return new ContextValue(LispTree.proto.parseFromString(str));
}
}

71
src/edu/stanford/nlp/sempre/Dataset.java
View File

@ -9,6 +9,7 @@ import fig.basic.*;
import fig.exec.Execution;
import fig.prob.SampleUtils;
import java.io.*;
import java.util.*;
/**
@ -35,9 +36,6 @@ public class Dataset {
@Option(gloss = "Only keep examples which have at most this number of tokens")
public int maxTokens = Integer.MAX_VALUE;
@Option(gloss = "Read dataset in full lisptree format (otherwise JSON).")
public boolean readLispTreeFormat = false;
}
public static Options opts = new Options();
@ -89,18 +87,25 @@ public class Dataset {
}
public void readFromPathPairs(List<Pair<String, String>> pathPairs) {
if (opts.readLispTreeFormat) {
readLispTreeFromPathPairs(pathPairs);
return;
// Try to detect whether we need JSON.
for (Pair<String, String> pathPair : pathPairs) {
if (pathPair.getSecond().endsWith(".json")) {
readJsonFromPathPairs(pathPairs);
return;
}
}
readLispTreeFromPathPairs(pathPairs);
}
private void readJsonFromPathPairs(List<Pair<String, String>> pathPairs) {
List<GroupInfo> groups = Lists.newArrayListWithCapacity(pathPairs.size());
for (Pair<String, String> pathPair : pathPairs) {
String group = pathPair.getFirst();
String path = pathPair.getSecond();
List<Example> examples = Json.readValueHard(
IOUtils.openInHard(path),
new TypeReference<List<Example>>() {});
new TypeReference<List<Example>>() { });
GroupInfo gi = new GroupInfo(group, examples);
gi.path = path;
groups.add(gi);
@ -123,7 +128,7 @@ public class Dataset {
LogInfo.end_track();
}
private void splitDevFromTrain() {
// Split original training examples randomly into train and dev.
List<Example> origTrainExamples = allExamples.get("train");
@ -143,8 +148,6 @@ public class Dataset {
devExamples.add(origTrainExamples.get(perm[i]));
}
}
private void readHelper(List<Example> incoming,
int maxExamples,
@ -162,7 +165,7 @@ public class Dataset {
ex = new Example.Builder().withExample(ex).setId(id).createExample();
}
i++;
ex.preprocess();
ex.preprocess(LanguageAnalyzer.getSingleton());
// Skip example if too long
if (ex.numTokens() > opts.maxTokens) continue;
@ -176,27 +179,8 @@ public class Dataset {
}
}
private void collectStats() {
LogInfo.begin_track_printAll("Dataset stats");
Execution.putLogRec("numTokenTypes", tokenTypes.size());
Execution.putLogRec("numTokensPerExample", numTokensFig);
for (Map.Entry<String, List<Example>> e : allExamples.entrySet())
Execution.putLogRec("numExamples." + e.getKey(), e.getValue().size());
LogInfo.end_track();
}
/**
* For reading datasets entirely in lisptree format.
*/
@Deprecated
public void readLispTree() {
readLispTreeFromPathPairs(opts.inPaths);
}
@Deprecated
private void readLispTreeFromPathPairs(List<Pair<String, String>> pathPairs) {
LogInfo.begin_track_printAll("Dataset.read");
for (Pair<String, String> pathPair : pathPairs) {
String group = pathPair.getFirst();
String path = pathPair.getSecond();
@ -210,7 +194,6 @@ public class Dataset {
LogInfo.end_track();
}
@Deprecated
private void readLispTreeHelper(String path, int maxExamples, List<Example> examples) {
if (examples.size() >= maxExamples) return;
LogInfo.begin_track("Reading %s", path);
@ -225,7 +208,7 @@ public class Dataset {
Example ex = Example.fromLispTree(tree, path + ":" + n); // Specify a default id if it doesn't exist
n++;
ex.preprocess();
ex.preprocess(LanguageAnalyzer.getSingleton());
// Skip example if too long
if (ex.numTokens() > opts.maxTokens) continue;
@ -239,6 +222,15 @@ public class Dataset {
LogInfo.end_track();
}
private void collectStats() {
LogInfo.begin_track_printAll("Dataset stats");
Execution.putLogRec("numTokenTypes", tokenTypes.size());
Execution.putLogRec("numTokensPerExample", numTokensFig);
for (Map.Entry<String, List<Example>> e : allExamples.entrySet())
Execution.putLogRec("numExamples." + e.getKey(), e.getValue().size());
LogInfo.end_track();
}
private static int getMaxExamplesForGroup(String group) {
int maxExamples = Integer.MAX_VALUE;
for (Pair<String, Integer> maxPair : opts.maxExamples)
@ -246,4 +238,19 @@ public class Dataset {
maxExamples = maxPair.getSecond();
return maxExamples;
}
public static void appendExampleToFile(String path, Example ex) {
// JSON is an annoying format because we can't just append.
// So currently we have to read the entire file in and write it out.
List<Example> examples;
if (new File(path).exists()) {
examples = Json.readValueHard(
IOUtils.openInHard(path),
new TypeReference<List<Example>>() { });
} else {
examples = new ArrayList<Example>();
}
examples.add(ex);
Json.prettyWriteValueHard(new File(path), examples);
}
}

46
src/edu/stanford/nlp/sempre/DateFn.java
View File

@ -1,39 +1,27 @@
package edu.stanford.nlp.sempre;
import java.util.Collections;
import java.util.List;
/**
* Maps a string to a Date.
*
* @author Percy Liang
*/
public class DateFn extends SemanticFn {
@Deprecated public static List<String> dateAnnotations(String s) {
return Collections.emptyList();
}
@Deprecated public static String dateAnnotation(String s) {
throw new RuntimeException("Remove me");
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
return o != null && getClass() == o.getClass();
}
public List<Derivation> call(Example ex, Callable c) {
String value = ex.languageInfo.getNormalizedNerSpan("DATE", c.getStart(), c.getEnd());
if (value == null) return Collections.emptyList();
DateValue dateValue = DateValue.parseDateValue(value);
if (dateValue == null) return Collections.emptyList();
return Collections.singletonList(
new Derivation.Builder()
.withCallable(c)
.formula(new ValueFormula<DateValue>(dateValue))
.type(SemType.dateType)
.createDerivation());
public DerivationStream call(final Example ex, final Callable c) {
return new SingleDerivationStream() {
@Override
public Derivation createDerivation() {
String value = ex.languageInfo.getNormalizedNerSpan("DATE", c.getStart(), c.getEnd());
if (value == null)
return null;
DateValue dateValue = DateValue.parseDateValue(value);
if (dateValue == null)
return null;
return new Derivation.Builder()
.withCallable(c)
.formula(new ValueFormula<>(dateValue))
.type(SemType.dateType)
.createDerivation();
}
};
}
}

38
src/edu/stanford/nlp/sempre/DateValue.java
View File

@ -1,6 +1,7 @@
package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.Calendar;
public class DateValue extends Value {
public final int year;
@ -19,7 +20,16 @@ public class DateValue extends Value {
int year = -1, month = -1, day = -1;
boolean isBC = dateStr.startsWith("-");
if (isBC) dateStr = dateStr.substring(1);
String dateParts[];
// Ignore time
int t = dateStr.indexOf('T');
if (t != -1) dateStr = dateStr.substring(0, t);
String[] dateParts;
if (dateStr.indexOf('T') != -1)
dateStr = dateStr.substring(0, dateStr.indexOf('T'));
dateParts = dateStr.split("-");
if (dateParts.length > 3)
throw new RuntimeException("Date has more than 3 parts: " + dateStr);
@ -41,6 +51,14 @@ public class DateValue extends Value {
return val;
}
public static DateValue now() {
Calendar cal = Calendar.getInstance();
int year = cal.get(Calendar.YEAR);
int month = cal.get(Calendar.MONTH);
int day = cal.get(Calendar.DAY_OF_MONTH);
return new DateValue(year, month, day);
}
public DateValue(int year, int month, int day) {
this.year = year;
this.month = month;
@ -62,17 +80,6 @@ public class DateValue extends Value {
return tree;
}
public double getCompatibility(Value thatValue) {
if (!(thatValue instanceof DateValue))
return 0;
DateValue that = (DateValue) thatValue;
// TODO: Only comparing the year right now.
boolean perfectMatch = (this.year == that.year);
//&& (this.month == that.month)
//&& (this.day == that.day);
return perfectMatch ? 1.0 : 0.0;
}
@Override public int hashCode() {
int hash = 0x7ed55d16;
hash = hash * 0xd3a2646c + year;
@ -81,9 +88,10 @@ public class DateValue extends Value {
return hash;
}
@Override public boolean equals(Object thatObj) {
if (!(thatObj instanceof DateValue)) return false;
DateValue that = (DateValue)thatObj;
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DateValue that = (DateValue) o;
if (this.year != that.year) return false;
if (this.month != that.month) return false;
if (this.day != that.day) return false;

58
src/edu/stanford/nlp/sempre/DerivOpCountFeatureComputer.java Normal file
View File

@ -0,0 +1,58 @@
package edu.stanford.nlp.sempre;
import java.util.*;
import com.google.common.collect.Sets;
import fig.basic.*;
/**
* Extracts indicator features that count how many times semantic functions and
* LHSs have been used in the derivation For now we count how many times MergeFn
* and JoinFn, also how many time unary, binary and entity lexical entries have
* been used. The feature is a pair with the operation and the count
*
* @author jonathanberant
*/
public class DerivOpCountFeatureComputer implements FeatureComputer {
public static class Options {
@Option(gloss = "Count only basic categories and SemanticFns")
public boolean countBasicOnly = true;
}
public static Options opts = new Options();
public static final String entityCat = "$Entity";
public static final String unaryCat = "$Unary";
public static final String binaryCat = "$Binary";
public static final String joinFn = "JoinFn";
public static final String mergeFn = "MergeFn";
public static final String bridgeFn = "BridgeFn";
public static Set<String> featureNames = Sets.newHashSet(entityCat, unaryCat, binaryCat, joinFn, mergeFn, bridgeFn);
@Override
public void extractLocal(Example ex, Derivation deriv) {
if (!FeatureExtractor.containsDomain("opCount")) return;
if (!deriv.isRoot(ex.numTokens())) return;
// extract the operation count
Map<String, Integer> opCounter = new HashMap<>();
extractOperationsRecurse(deriv, opCounter);
addFeatures(deriv, opCounter);
}
private void extractOperationsRecurse(Derivation deriv, Map<String, Integer> opCounter) {
// Basic case: no rule
if (deriv.children.isEmpty()) return;
// increment counts for current rule
MapUtils.incr(opCounter, deriv.rule.lhs);
MapUtils.incr(opCounter, deriv.rule.sem.getClass().getSimpleName());
// recursive call
for (Derivation child : deriv.children)
extractOperationsRecurse(child, opCounter);
}
private void addFeatures(Derivation deriv, Map<String, Integer> opCounter) {
for (String feature : (opts.countBasicOnly ? featureNames : opCounter.keySet()))
deriv.addFeature("opCount", "count(" + feature + ")=" + MapUtils.get(opCounter, feature, 0));
}
}

81
src/edu/stanford/nlp/sempre/DerivOpCountFeatureExtractor.java
View File

@ -1,81 +0,0 @@
package edu.stanford.nlp.sempre;
import edu.stanford.nlp.stats.ClassicCounter;
import edu.stanford.nlp.stats.Counter;
import edu.stanford.nlp.util.ArrayUtils;
import java.util.Set;
/**
* Extracts indicator features that count how many times semantic functions and
* LHSs have been used in the derivation For now we count how many times MergeFn
* and JoinFn, also how many time unary, binary and entity lexical entries have
* been used. The feature is a pair with the operation and the count
*
* @author jonathanberant
*/
public class DerivOpCountFeatureExtractor {
public static final String entityCat = "$Entity";
public static final String unaryCat = "$Unary";
public static final String binaryCat = "$Binary";
public final static String joinFn = "JoinFn";
public final static String mergeFn = "MergeFn";
public final static String bridgeFn = "BridgeFn";
public static Set<String> featureNames = ArrayUtils.asSet(new String[]{entityCat, unaryCat, binaryCat, joinFn, mergeFn, bridgeFn});
public void extractLocal(Example ex, Derivation deriv) {
if (!FeatureExtractor.containsDomain("opCount")) return;
if (!deriv.isRoot(ex.numTokens())) return;
//extract the operation count
Counter<String> opCounter = new ClassicCounter<String>();
extractOperationsRecurse(deriv, opCounter);
addFeatures(deriv, opCounter);
//add pre-terminal sequence
//StringBuilder sb = new StringBuilder();
//extractPreterminalYieldRecurse(deriv, sb);
//deriv.localFeatureVector.add(sb.toString());
}
private void extractPreterminalYieldRecurse(Derivation deriv, StringBuilder sb) {
//base case 1
if (deriv.children.size() == 0)
return;
//base case 2
if (deriv.rule.lhs.equals(entityCat) || deriv.rule.lhs.equals(unaryCat) || deriv.rule.lhs.equals(binaryCat)) {
sb.append(deriv.rule.lhs + "_");
return;
}
//recursive call from left to right
for (Derivation child : deriv.children) {
extractPreterminalYieldRecurse(child, sb);
}
}
private void extractOperationsRecurse(Derivation deriv, Counter<String> opCounter) {
//base case
if (deriv.children.size() == 0)
return;
//boolean[] appendPreTerminal = new boolean[deriv.children.size()];
//increment counts for current rule
opCounter.incrementCount(deriv.rule.lhs);
if (deriv.rule.sem instanceof JoinFn)
opCounter.incrementCount(joinFn);
else if (deriv.rule.sem instanceof MergeFn)
opCounter.incrementCount(mergeFn);
else if (deriv.rule.sem instanceof BridgeFn)
opCounter.incrementCount(bridgeFn);
//recursive call
for (Derivation child : deriv.children)
extractOperationsRecurse(child, opCounter);
}
private void addFeatures(Derivation deriv, Counter<String> opCounter) {
for (String feature : featureNames)
deriv.addFeature("opCount", "count(" + feature + ")=" + (int)opCounter.getCount(feature));
}
}

337
src/edu/stanford/nlp/sempre/Derivation.java
View File

@ -1,8 +1,5 @@
package edu.stanford.nlp.sempre;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import fig.basic.*;
import java.util.*;
@ -15,13 +12,18 @@ import java.util.*;
*
* @author Percy Liang
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Derivation implements SemanticFn.Callable {
public class Derivation implements SemanticFn.Callable, HasScore {
public static class Options {
@Option(gloss = "When printing derivations, to show values (could be quite verbose)")
public boolean showValues = true;
@Option(gloss = "When printing derivations, to show the first value (ignored when showValues is set)")
public boolean showFirstValue = false;
@Option(gloss = "When printing derivations, to show types")
public boolean showTypes = true;
@Option(gloss = "When printing derivations, to show rules")
public boolean showRules = false;
@Option(gloss = "When printing derivations, to show canonical utterance")
public boolean showUtterance = false;
}
public static Options opts = new Options();
@ -29,14 +31,13 @@ public class Derivation implements SemanticFn.Callable {
//// Basic fields: created by the constructor.
// Span that the derivation is built over
@JsonProperty public final String cat;
@JsonProperty public final int start;
@JsonProperty public final int end;
public final String cat;
public final int start;
public final int end;
public String canonicalUtterance;
// If this derivation is composed of other derivations
@JsonProperty
public final Rule rule; // Which rule was used to produce this derivation? Set to nullRule if not.
@JsonProperty
public final List<Derivation> children; // Corresponds to the RHS of the rule.
//// SemanticFn fields: read/written by SemanticFn.
@ -44,63 +45,46 @@ public class Derivation implements SemanticFn.Callable {
// information. This could be its own class, but expose more right now to
// be more flexible.
@JsonProperty
public final Formula formula; // Logical form produced by this derivation
@JsonProperty
public final SemType type; // Type corresponding to that logical form
//// Fields produced by feature extractor, evaluation, etc.
final List<String> localChoices = new ArrayList<String>(); // Just for printing/debugging.
private List<String> localChoices; // Just for printing/debugging.
// TODO(pliang): make fields private
// Information for scoring
@JsonProperty final private FeatureVector localFeatureVector; // Features
@JsonProperty double score = Double.NaN; // Weighted combination of features
private final FeatureVector localFeatureVector; // Features
double score = Double.NaN; // Weighted combination of features
// What the formula evaluates to (optionally set later; only non-null for the root Derivation)
@JsonProperty Value value;
@JsonProperty Evaluation executorStats;
public Value value;
public Evaluation executorStats;
// Number in [0, 1] denoting how correct the value is.
@JsonProperty double compatibility = Double.NaN;
public double compatibility = Double.NaN;
// Probability (normalized exp of score).
@JsonProperty double prob = Double.NaN;
public double prob = Double.NaN;
// Miscellaneous statistics which are filled in by BeamParser
// TODO: wrap this in an object BeamParserStats
// Miscellaneous statistics
int maxBeamPosition = -1; // Lowest position that this tree or any of its children is on the beam (after sorting)
int maxUnsortedBeamPosition = -1; // Lowest position that this tree or any of its children is on the beam (before sorting)
int preSortBeamPosition = -1;
int postSortBeamPosition = -1;
//caching the hashcode
// Cache the hash code
int hashCode = -1;
// Initially, derivation is created with these parameters.
// Then, SemanticFn is called to create many copies of this derivation with different
// Each derivation that gets created gets a unique ID in increasing order so that
// we can break ties consistently for reproducible results.
long creationIndex;
public static long numCreated = 0; // Incremented for each derivation we create.
public static final Comparator<Derivation> derivScoreComparator = new ScoredDerivationComparator();
// TODO: why do we need these constructors if we have the Builder?
/** Constructor for humans. */
public Derivation(String cat, int start, int end,
Rule rule,
List<Derivation> children,
Formula formula,
SemType type) {
this.cat = cat;
this.start = start;
this.end = end;
this.rule = rule;
this.children = children;
this.formula = formula;
this.type = type;
this.localFeatureVector = new FeatureVector();
}
public static final List<Derivation> emptyList = Collections.emptyList();
/** Constructor for humans. */
public Derivation(String cat, int start, int end,
Rule rule,
List<Derivation> children) {
this(cat, start, end, rule, children, null, null);
}
// A Derivation is built from
/** Builder for everyone. */
public static class Builder {
@ -117,61 +101,25 @@ public class Derivation implements SemanticFn.Callable {
private Evaluation executorStats;
private double compatibility = Double.NaN;
private double prob = Double.NaN;
private String canonicalUtterance = "";
public Builder cat(String cat) { this.cat = cat; return this; }
public Builder start(int start) { this.start = start; return this; }
public Builder end(int end) { this.end = end; return this; }
public Builder rule(Rule rule) { this.rule = rule; return this; }
public Builder children(List<Derivation> children) { this.children = children; return this; }
public Builder formula(Formula formula) { this.formula = formula; return this; }
public Builder type(SemType type) { this.type = type; return this; }
public Builder localFeatureVector(FeatureVector localFeatureVector) { this.localFeatureVector = localFeatureVector; return this; }
public Builder score(double score) { this.score = score; return this; }
public Builder value(Value value) { this.value = value; return this; }
public Builder executorStats(Evaluation executorStats) { this.executorStats = executorStats; return this; }
public Builder compatibility(double compatibility) { this.compatibility = compatibility; return this; }
public Builder prob(double prob) { this.prob = prob; return this; }
public Builder canonicalUtterance(String canonicalUtterance) { this.canonicalUtterance = canonicalUtterance; return this; }
public Builder cat(String cat) {
this.cat = cat;
return this;
}
public Builder start(int start) {
this.start = start;
return this;
}
public Builder end(int end) {
this.end = end;
return this;
}
public Builder rule(Rule rule) {
this.rule = rule;
return this;
}
public Builder children(List<Derivation> children) {
this.children = children;
return this;
}
public Builder formula(Formula formula) {
this.formula = formula;
return this;
}
public Builder type(SemType type) {
this.type = type;
return this;
}
public Builder localFeatureVector(FeatureVector localFeatureVector) {
this.localFeatureVector = localFeatureVector;
return this;
}
public Builder score(double score) {
this.score = score;
return this;
}
public Builder value(Value value) {
this.value = value;
return this;
}
public Builder executorStats(Evaluation executorStats) {
this.executorStats = executorStats;
return this;
}
public Builder compatibility(double compatibility) {
this.compatibility = compatibility;
return this;
}
public Builder prob(double prob) {
this.prob = prob;
return this;
}
public Builder withStringFormulaFrom(String value) {
this.formula = new ValueFormula<StringValue>(new StringValue(value));
this.formula = new ValueFormula<>(new StringValue(value));
this.type = SemType.stringType;
return this;
}
@ -180,6 +128,7 @@ public class Derivation implements SemanticFn.Callable {
this.type = deriv.type;
return this;
}
public Builder withCallable(SemanticFn.Callable c) {
this.cat = c.getCat();
this.start = c.getStart();
@ -188,27 +137,16 @@ public class Derivation implements SemanticFn.Callable {
this.children = c.getChildren();
return this;
}
public Derivation createDerivation() {
return new Derivation(
cat, start, end, rule, children, formula, type,
localFeatureVector, score, value, executorStats, compatibility, prob);
localFeatureVector, score, value, executorStats, compatibility, prob, canonicalUtterance);
}
}
@JsonCreator
Derivation(@JsonProperty("cat") String cat,
@JsonProperty("start") int start,
@JsonProperty("end") int end,
@JsonProperty("rule") Rule rule,
@JsonProperty("children") List<Derivation> children,
@JsonProperty("formula") Formula formula,
@JsonProperty("type") SemType type,
@JsonProperty("localFeatureVector") FeatureVector localFeatureVector,
@JsonProperty("score") double score,
@JsonProperty("value") Value value,
@JsonProperty("executorStats") Evaluation executorStats,
@JsonProperty("compatibility") double compatibility,
@JsonProperty("prob") double prob) {
Derivation(String cat, int start, int end, Rule rule, List<Derivation> children, Formula formula, SemType type,
FeatureVector localFeatureVector, double score, Value value, Evaluation executorStats, double compatibility, double prob, String canonicalUtterance) {
this.cat = cat;
this.start = start;
this.end = end;
@ -222,6 +160,8 @@ public class Derivation implements SemanticFn.Callable {
this.executorStats = executorStats;
this.compatibility = compatibility;
this.prob = prob;
this.canonicalUtterance = canonicalUtterance;
this.creationIndex = numCreated++;
}
public Formula getFormula() { return formula; }
@ -230,11 +170,14 @@ public class Derivation implements SemanticFn.Callable {
public double getCompatibility() { return compatibility; }
public List<Derivation> getChildren() { return children; }
public Value getValue() { return value; }
public boolean isFeaturizedAndScored() { return !Double.isNaN(score); }
public boolean isExecuted() { return value != null; }
public int getMaxBeamPosition() { return maxBeamPosition; }
public String getCat() { return cat; }
public int getStart() { return start; }
public int getEnd() { return end; }
public boolean containsIndex(int i) { return i < end && i >= start; }
public Rule getRule() { return rule; }
public Evaluation getExecutorStats() { return executorStats; }
@ -243,24 +186,31 @@ public class Derivation implements SemanticFn.Callable {
return Formulas.getString(children.get(i).formula);
}
// Return whether |deriv| is the top Derivation.
// Return whether |deriv| is built over the root Derivation.
public boolean isRoot(int numTokens) {
return cat.equals(Rule.rootCat) && start == 0 && end == numTokens;
return cat.equals(Rule.rootCat) && ((start == 0 && end == numTokens) || (start == -1));
}
// Functions that operate on features.
public void addFeature(String domain, String name) { addFeature(domain, name, 1); }
public void addFeature(String domain, String name, double value) { this.localFeatureVector.add(domain, name, value); }
public void addHistogramFeature(String domain, String name, double value,
int initBinSize, int numBins, boolean exp) {
this.localFeatureVector.addHistogram(domain, name, value, initBinSize, numBins, exp);
}
public void addFeatureWithBias(String domain, String name, double value) { this.localFeatureVector.addWithBias(domain, name, value); }
public void addFeatures(FeatureVector fv) { this.localFeatureVector.add(fv); }
public void addFeatures(Derivation deriv) { this.localFeatureVector.add(deriv.localFeatureVector); }
public double localScore(Params params) {
return localFeatureVector.dotProduct(params);
}
/**
* Recursively compute the score for each node in derivation. Update |score|
* field as well as return its value.
*/
public double computeScore(Params params) {
score = localFeatureVector.dotProduct(params);
score = localScore(params);
if (children != null)
for (Derivation child : children)
score += child.computeScore(params);
@ -272,94 +222,63 @@ public class Derivation implements SemanticFn.Callable {
* already scored).
*/
public double computeScoreLocal(Params params) {
score = localFeatureVector.dotProduct(params);
score = localScore(params);
if (children != null)
for (Derivation child : children)
score += child.score;
return score;
}
public void ensureExecuted(Executor executor) {
if (!isExecuted()) {
StopWatchSet.begin("Executor.execute");
Executor.Response response = executor.execute(formula);
StopWatchSet.end();
value = response.value;
executorStats = response.stats;
}
}
/** Copy execution result and stats from another Derivation. */
public void setExecResults(Derivation deriv) {
if (isExecuted())
throw new IllegalStateException("setExecResults() on an executed Derivation");
value = deriv.value;
executorStats = deriv.executorStats;
// TODO: why do we need this?
localFeatureVector.add(deriv.localFeatureVector, DenotationFeatureMatcher.matcher);
}
@Override
public boolean equals(Object thatObj) {
//if(thatObj == null || getClass() != thatObj.getClass()) return false;
Derivation that = (Derivation) thatObj;
if (!this.cat.equals(that.cat)) return false;
if (this.start != that.start) return false;
if (this.end != that.end) return false;
if (!this.rule.equals(that.rule)) return false;
if (!this.children.equals(that.children)) return false;
if (!this.formula.equals(that.formula)) return false;
return true;
}
@Override
public int hashCode() {
if(hashCode==-1) {
int hash = 0x7ed55d16;
hash = hash * 0xd3a2646c + cat.hashCode();
hash = hash * 0xd3a2646c + start;
hash = hash * 0xd3a2646c + end;
hash = hash * 0xd3a2646c + rule.hashCode();
hash = hash * 0xd3a2646c + children.hashCode();
hash = hash * 0xd3a2646c + formula.hashCode();
// Boolean b = isCompleteDerivation();
// hash = hash * 0xd3a2646c + b.hashCode();
hashCode = hash;
}
return hashCode;
// If we haven't executed the formula associated with this derivation, then
// execute it!
public void ensureExecuted(Executor executor, ContextValue context) {
if (isExecuted()) return;
StopWatchSet.begin("Executor.execute");
Executor.Response response = executor.execute(formula, context);
StopWatchSet.end();
value = response.value;
executorStats = response.stats;
}
public LispTree toLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild("derivation");
//tree.addChild(LispTree.proto.newList("span", cat+"["+start+":"+end+"]"));
if (formula != null)
tree.addChild(LispTree.proto.newList("formula", formula.toLispTree()));
if (value != null) {
if (opts.showValues)
tree.addChild(LispTree.proto.newList("value", value.toLispTree()));
else if (value instanceof ListValue)
tree.addChild(((ListValue) value).values.size() + " values");
else if (value instanceof ListValue) {
List<Value> values = ((ListValue) value).values;
if (opts.showFirstValue && values.size() > 0) {
tree.addChild(LispTree.proto.newList(values.size() + " values", values.get(0).toLispTree()));
} else {
tree.addChild(values.size() + " values");
}
}
}
if (type != null)
if (type != null && opts.showTypes)
tree.addChild(LispTree.proto.newList("type", type.toLispTree()));
if (opts.showRules) {
if (rule != null) tree.addChild(getRuleLispTree());
}
if (opts.showUtterance && canonicalUtterance != null) {
tree.addChild(LispTree.proto.newList("canonicalUtterance", canonicalUtterance));
}
return tree;
}
/**
* lisp tree showing the entire parse tree
* @return
* @return lisp tree showing the entire parse tree
*/
public LispTree toRecursiveLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild("derivation");
tree.addChild(LispTree.proto.newList("span", cat+"["+start+":"+end+"]"));
tree.addChild(LispTree.proto.newList("span", cat + "[" + start + ":" + end + "]"));
if (formula != null)
tree.addChild(LispTree.proto.newList("formula", formula.toLispTree()));
for(Derivation child: children)
for (Derivation child : children)
tree.addChild(child.toRecursiveLispTree());
return tree;
}
@ -368,13 +287,13 @@ public class Derivation implements SemanticFn.Callable {
return toRecursiveLispTree().toString();
}
// TODO(pliang): remove this in favor of localChoices
private LispTree getRuleLispTree() {
LispTree tree = LispTree.proto.newList();
tree.addChild("rules");
getRuleLispTreeRecurs(tree);
return tree;
}
private void getRuleLispTreeRecurs(LispTree tree) {
if (children.size() > 0) {
tree.addChild(LispTree.proto.newList("rule", rule.toLispTree()));
@ -389,6 +308,9 @@ public class Derivation implements SemanticFn.Callable {
}
public String toString() { return toLispTree().toString(); }
public void incrementLocalFeatureVector(double factor, Map<String, Double> map) {
localFeatureVector.increment(factor, map, AllFeatureMatcher.matcher);
}
public void incrementAllFeatureVector(double factor, Map<String, Double> map) {
incrementAllFeatureVector(factor, map, AllFeatureMatcher.matcher);
}
@ -398,35 +320,41 @@ public class Derivation implements SemanticFn.Callable {
child.incrementAllFeatureVector(factor, map, updateFeatureMatcher);
}
// recursively renames all features in derivation by adding a prefix
public FeatureVector addPrefixLocalFeatureVector(String prefix) {
return localFeatureVector.addPrefix(prefix);
}
public Map<String, Double> getAllFeatureVector() {
Map<String, Double> m = new HashMap<String, Double>();
Map<String, Double> m = new HashMap<>();
incrementAllFeatureVector(1.0d, m, AllFeatureMatcher.matcher);
return m;
}
// TODO: this is crazy inefficient
public Double getAllFeatureVector(String featureName) {
Map<String, Double> m = new HashMap<String, Double>();
// TODO(pliang): this is crazy inefficient
public double getAllFeatureVector(String featureName) {
Map<String, Double> m = new HashMap<>();
incrementAllFeatureVector(1.0d, m, new ExactFeatureMatcher(featureName));
return MapUtils.get(m,featureName,0.0);
return MapUtils.get(m, featureName, 0.0);
}
public void addLocalChoice(String choice) {
if (localChoices == null)
localChoices = new ArrayList<String>();
localChoices.add(choice);
}
public void incrementAllChoices(int factor, Map<String, Integer> map) {
if (opts.showRules)
MapUtils.incr(map, "[" + start + ":" + end + "] " + rule.toString(), 1);
for (String choice : localChoices)
MapUtils.incr(map, choice, factor);
if (localChoices != null) {
for (String choice : localChoices)
MapUtils.incr(map, choice, factor);
}
for (Derivation child : children)
child.incrementAllChoices(factor, map);
}
//methods added to allow checking if a derivation is complete and completing it
public boolean isCompleteDerivation() {return true;}
public List<Derivation> complete(Example ex) {
throw new RuntimeException("Can not complete a derivation that is already complete, use isCompleteDerivation()");
}
// Used to compare derivations by score.
public static class ScoredDerivationComparator implements Comparator<Derivation> {
@Override
@ -434,14 +362,37 @@ public class Derivation implements SemanticFn.Callable {
if (deriv1.score > deriv2.score) return -1;
if (deriv1.score < deriv2.score) return +1;
// Ensure reproducible randomness
if (deriv1.hashCode() < deriv2.hashCode()) return -1;
if (deriv1.hashCode() > deriv2.hashCode()) return +1;
if (deriv1.creationIndex < deriv2.creationIndex) return -1;
if (deriv1.creationIndex > deriv2.creationIndex) return +1;
return 0;
}
}
// Used to compare derivations by compatibility.
public static class CompatibilityDerivationComparator implements Comparator<Derivation> {
@Override
public int compare(Derivation deriv1, Derivation deriv2) {
if (deriv1.compatibility > deriv2.compatibility) return -1;
if (deriv1.compatibility < deriv2.compatibility) return +1;
// Ensure reproducible randomness
if (deriv1.creationIndex < deriv2.creationIndex) return -1;
if (deriv1.creationIndex > deriv2.creationIndex) return +1;
return 0;
}
}
// for debugging
public void printDerivationRecursively() {
LogInfo.logs("Deriv: %s(%s,%s) %s", cat, start, end, formula);
for (int i = 0; i < children.size(); i++) {
LogInfo.begin_track("child %s:", i);
children.get(i).printDerivationRecursively();
LogInfo.end_track();
}
}
public static void sortByScore(List<Derivation> trees) {
Collections.sort(trees, new ScoredDerivationComparator());
Collections.sort(trees, derivScoreComparator);
}
// Generate a probability distribution over derivations given their scores.

28
src/edu/stanford/nlp/sempre/DerivationConstraint.java
View File

@ -1,28 +0,0 @@
package edu.stanford.nlp.sempre;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
/**
* A DerivationConstraint filters the set of Derivations. Currently, it's just
* used to visualize the Derivations.
*/
public class DerivationConstraint {
// Regular expression on formula's toString()
private final String formulaPattern;
@JsonCreator
public DerivationConstraint(String formulaPattern) {
this.formulaPattern = formulaPattern;
}
// Intended just for JsonValue.
@JsonValue
public String getFormulaPattern() { return formulaPattern; }
// Return the factor
public boolean satisfies(Example ex, Derivation deriv) {
//LogInfo.logs("satisfies: %s %s", deriv, formulaPattern);
return deriv.getFormula().toString().matches(".*" + formulaPattern + ".*");
}
}

13
src/edu/stanford/nlp/sempre/DerivationStream.java Normal file
View File

@ -0,0 +1,13 @@
package edu.stanford.nlp.sempre;
import java.util.Iterator;
/**
* Represents a stream of Derivations which are constructed lazily for efficiency.
* Use either SingleDerivationStream or MultipleDerivationStream.
* Created by joberant on 3/14/14.
*/
public interface DerivationStream extends Iterator<Derivation> {
Derivation peek();
int estimatedSize();
}

21
src/edu/stanford/nlp/sempre/DescriptionValue.java
View File

@ -1,7 +1,6 @@
package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import fig.basic.Option;
/**
* Represents the description part of a NameValue ("Barack Obama" rather than
@ -10,11 +9,6 @@ import fig.basic.Option;
* @author Andrew Chou
*/
public class DescriptionValue extends Value {
public static class Options {
@Option(gloss = "Verbose.") public boolean verbose = false;
}
public static Options opts = new Options();
public final String value;
public DescriptionValue(LispTree tree) { this(tree.child(1).value); }
@ -27,18 +21,11 @@ public class DescriptionValue extends Value {
return tree;
}
public double getCompatibility(Value thatValue) {
// Match the description part of NameValue.
if (thatValue instanceof NameValue)
return value.equals(((NameValue)thatValue).description) ? 1 : 0;
return super.getCompatibility(thatValue);
}
@Override public int hashCode() { return value.hashCode(); }
@Override public boolean equals(Object thatObj) {
if (!(thatObj instanceof DescriptionValue)) return false;
DescriptionValue that = (DescriptionValue)thatObj;
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DescriptionValue that = (DescriptionValue) o;
return this.value.equals(that.value);
}
}

8
src/edu/stanford/nlp/sempre/ErrorValue.java
View File

@ -31,6 +31,9 @@ public class ErrorValue extends Value {
// Server returned something back but it had a bad format (e.g., HTML instead of XML).
public static final ErrorValue badFormat = new ErrorValue("BADFORMAT");
// Execution of Java failed (generated by JavaExecutor).
public static final ErrorValue badJava(String message) { return new ErrorValue("BADJAVA: " + message); }
public final String type;
public ErrorValue(LispTree tree) { this.type = tree.child(1).value; }
@ -44,6 +47,7 @@ public class ErrorValue extends Value {
}
@Override
// TODO(pliang): return this (error type) to avoid clashes with NameValue
public String toString() { return type; }
public static ErrorValue fromString(String s) {
if (s.equals(timeout.type)) return timeout;
@ -54,10 +58,6 @@ public class ErrorValue extends Value {
return null;
}
public double getCompatibility(Value thatValue) {
return 0; // Never give points for error.
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

130
src/edu/stanford/nlp/sempre/Evaluation.java
View File

@ -1,130 +0,0 @@
package edu.stanford.nlp.sempre;
import com.fasterxml.jackson.annotation.JsonValue;
import com.google.common.collect.Maps;
import fig.basic.Fmt;
import fig.basic.LispTree;
import fig.basic.LogInfo;
import fig.basic.StatFig;
import fig.exec.Execution;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* An Evaluation measures how well the system is doing on a set of examples.
* Formally, it is just a collection of arbitrary statistics.
*
* @author Percy Liang
*/
public class Evaluation {
private List<String> names = new ArrayList<String>();
private List<StatFig> values = new ArrayList<StatFig>();
public StatFig getFig(String name) {
int i = names.indexOf(name);
if (i == -1)
return null;
return values.get(i);
}
// Methods to add new metrics to evaluation and aggregate evaluations.
private StatFig getFigHard(String name) {
int i = names.indexOf(name);
if (i == -1) {
i = names.size();
names.add(name);
values.add(new StatFig());
}
return values.get(i);
}
public void add(String name, boolean value) { add(name, value ? 1 : 0); }
public void add(String name, double value) { add(name, null, value); }
public void add(String name, Object key, double value) {
StatFig fig = new StatFig();
fig.add(key, value);
add(name, fig);
}
public void add(String name, StatFig fig) {
getFigHard(name).add(fig);
}
public synchronized void add(Evaluation eval) {
for (int i = 0; i < eval.names.size(); i++)
add(eval.names.get(i), eval.values.get(i));
}
@Deprecated
public LispTree toLispTree() {
LispTree out = LispTree.proto.newList();
for (int i = 0; i < names.size(); i++) {
out.addChild(LispTree.proto.newList(names.get(i), Fmt.D(values.get(i).mean())));
}
return out;
}
@Deprecated
public static Evaluation fromLispTree(LispTree t) {
Evaluation e = new Evaluation();
for (int i = 0; i < t.children.size(); i++) {
e.add(
t.child(i).child(0).value,
Double.parseDouble(t.child(i).child(1).value));
}
return e;
}
public String summary() { return summary(" "); }
public String summary(String delim) {
StringBuilder out = new StringBuilder();
for (int i = 0; i < names.size(); i++) {
if (i > 0) out.append(delim);
out.append(names.get(i) + '=' + Fmt.D(values.get(i).mean()));
}
return out.toString();
}
private void putOutput(String prefix, StatFig fig) {
Execution.putOutput(prefix + ".count", fig.count());
if (fig.count() > 0) {
Execution.putOutput(prefix + ".mean", fig.mean());
Execution.putOutput(prefix + ".max", fig.max());
}
}
private String basePrefix(String prefix) {
String[] parts = prefix.split("\\."); return parts[parts.length - 1];
}
public void putOutput(String prefix) {
for (int i = 0; i < names.size(); i++)
putOutput(basePrefix(prefix) + "." + names.get(i), values.get(i));
}
public void logStats(String prefix) {
LogInfo.begin_track_printAll("Evaluation stats for %s", prefix);
for (int i = 0; i < names.size(); i++)
LogInfo.log(names.get(i) + " = " + values.get(i));
LogInfo.end_track();
}
@JsonValue
Map<String, Map<String, Object>> toMap() {
Map<String, Map<String, Object>> m = Maps.newHashMapWithExpectedSize(names.size());
for (int i = 0; i < names.size(); i++)
m.put(names.get(i), statFigToMap(values.get(i)));
return m;
}
private static Map<String, Object> statFigToMap(StatFig fig) {
Map<String, Object> m = Maps.newHashMapWithExpectedSize(7);
m.put("min", fig.min());
m.put("minKey", fig.minKey());
m.put("max", fig.max());
m.put("maxKey", fig.maxKey());
m.put("mean", fig.mean());
m.put("stddev", fig.stddev());
m.put("count", fig.count());
return m;
}
}

8
src/edu/stanford/nlp/sempre/ExactValueEvaluator.java Normal file
View File

@ -0,0 +1,8 @@
package edu.stanford.nlp.sempre;
// This is the simplest evaluator, but exact match can sometimes be too harsh.
public class ExactValueEvaluator implements ValueEvaluator {
public double getCompatibility(Value target, Value pred) {
return target.equals(pred) ? 1 : 0;
}
}

201
src/edu/stanford/nlp/sempre/Example.java
View File

@ -1,8 +1,12 @@
package edu.stanford.nlp.sempre;
import com.fasterxml.jackson.annotation.*;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import fig.basic.Evaluation;
import fig.basic.LispTree;
import fig.basic.LogInfo;
@ -18,14 +22,9 @@ import java.util.List;
* @author Percy Liang
* @author Roy Frostig
*/
@JsonIgnoreProperties(ignoreUnknown=true)
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Example {
public static class JsonViews {
public static class WithDerivations {}
public static class WithDPChart {}
}
//// Information from the input file.
// Unique identifier for this example.
@ -34,181 +33,96 @@ public class Example {
// Input utterance
@JsonProperty public final String utterance;
// Provides
@JsonProperty public final DerivationConstraint derivConstraint;
// Context
@JsonProperty public ContextValue context;
// What we should try to predict.
@JsonProperty public Formula targetFormula; // Logical form
@JsonProperty public Value targetValue; // Answer
@JsonProperty public Formula targetFormula; // Logical form (e.g., database query)
@JsonProperty public Value targetValue; // Denotation (e.g., answer)
//// Information after preprocessing (e.g., tokenization, POS tagging, NER, syntactic parsing, etc.).
@JsonProperty public LanguageInfo languageInfo = null;
// Tokens come from languageInfo, but if we don't have one,
// they go here (usually due to deserialization).
// DELETE
private List<String> backupTokens = null;
//// Output of the parser.
// Predicted derivations: sorted by score.
@JsonProperty @JsonView(JsonViews.WithDerivations.class)
List<Derivation> predDerivations;
// Predicted derivations (sorted by score).
public List<Derivation> predDerivations;
// To debug amount of ordering change due to executing and obtaining
// denotation features.
List<Derivation> predDerivationsAfterParse;
// Statistics about how well we did during parsing and execution.
private Evaluation parseEvaluation;
private Evaluation evaluation;
// Maximum position in any cell of the chart of any sub-derivation of a
// correct derivation. Under the current parameters, we would need to set
// the beam size to at least this to get it right.
int correctMaxBeamPosition = -1;
// Beam size to use for parsing
int beamSize = -1;
// Statistics relating to processing the example.
public Evaluation evaluation;
public static class Builder {
private String id;
private String utterance;
private DerivationConstraint derivConstraint;
private ContextValue context;
private Formula targetFormula;
private Value targetValue;
private List<Derivation> predDerivations;
private LanguageInfo languageInfo;
public Builder setId(String id) {
this.id = id;
return this;
}
public Builder setUtterance(String utterance) {
this.utterance = utterance;
return this;
}
public Builder setDerivConstraint(DerivationConstraint derivConstraint) {
this.derivConstraint = derivConstraint;
return this;
}
public Builder setTargetFormula(Formula targetFormula) {
this.targetFormula = targetFormula;
return this;
}
public Builder setTargetValue(Value targetValue) {
this.targetValue = targetValue;
return this;
}
public Builder setPredDerivations(List<Derivation> predDerivations) {
this.predDerivations = predDerivations;
return this;
}
public Builder setLanguageInfo(LanguageInfo languageInfo) {
this.languageInfo = languageInfo;
return this;
}
public Builder setId(String id) { this.id = id; return this; }
public Builder setUtterance(String utterance) { this.utterance = utterance; return this; }
public Builder setContext(ContextValue context) { this.context = context; return this; }
public Builder setTargetFormula(Formula targetFormula) { this.targetFormula = targetFormula; return this; }
public Builder setTargetValue(Value targetValue) { this.targetValue = targetValue; return this; }
public Builder setLanguageInfo(LanguageInfo languageInfo) { this.languageInfo = languageInfo; return this; }
public Builder withExample(Example ex) {
setId(ex.id);
setUtterance(ex.utterance);
setDerivConstraint(ex.derivConstraint);
setContext(ex.context);
setTargetFormula(ex.targetFormula);
setTargetValue(ex.targetValue);
setPredDerivations(ex.predDerivations);
return this;
}
public Example createExample() {
return new Example(
id, utterance, derivConstraint, targetFormula,
targetValue, predDerivations, languageInfo);
return new Example(id, utterance, context, targetFormula, targetValue, languageInfo);
}
}
@JsonCreator
public Example(@JsonProperty("id") String id,
@JsonProperty("utterance") String utterance,
@JsonProperty("derivConstraint") DerivationConstraint derivConstraint,
@JsonProperty("context") ContextValue context,
@JsonProperty("targetFormula") Formula targetFormula,
@JsonProperty("targetValue") Value targetValue,
@JsonProperty("predDerivations") List<Derivation> predDerivations,
@JsonProperty("languageInfo") LanguageInfo languageInfo) {
this.id = id;
this.utterance = utterance;
this.derivConstraint = derivConstraint;
this.context = context;
this.targetFormula = targetFormula;
this.targetValue = targetValue;
this.predDerivations = predDerivations;
this.languageInfo = languageInfo;
}
// Accessors
public String getId() { return id; }
public String getUtterance() { return utterance; }
public Evaluation getEvaluation() { return evaluation; }
public int numTokens() { return languageInfo.tokens.size(); }
public List<Derivation> getPredDerivations() { return predDerivations; }
public void setTargetFormula(Formula targetFormula) {
this.targetFormula = targetFormula;
}
public void setTargetValue(Value targetValue) {
this.targetValue = targetValue;
}
public void setContext(ContextValue context) { this.context = context; }
public void setTargetFormula(Formula targetFormula) { this.targetFormula = targetFormula; }
public void setTargetValue(Value targetValue) { this.targetValue = targetValue; }
public String spanString(int start, int end) {
return String.format("%d:%d[%s]", start, end, phraseString(start, end));
return String.format("%d:%d[%s]", start, end, start != -1 ? phraseString(start, end) : "...");
}
public String phraseString(int start, int end) {
return Joiner.on(' ').join(languageInfo.tokens.subList(start, end));
}
// Return a string representing the tokens between start and end.
public List<String> getTokens() {
return (languageInfo != null) ? languageInfo.tokens : backupTokens;
}
public List<String> getTokens() { return languageInfo.tokens; }
public List<String> getLemmaTokens() { return languageInfo.lemmaTokens; }
public String token(int i) { return languageInfo.tokens.get(i); }
public String lemmaToken(int i) { return languageInfo.lemmaTokens.get(i); }
public String posTag(int i) { return languageInfo.posTags.get(i); }
public String phrase(int start, int end) {
return languageInfo.phrase(start, end);
}
public String lemmaPhrase(int start, int end) {
return languageInfo.lemmaPhrase(start, end);
}
public String phrase(int start, int end) { return languageInfo.phrase(start, end); }
public String lemmaPhrase(int start, int end) { return languageInfo.lemmaPhrase(start, end); }
void setParseEvaluation(Evaluation eval) { parseEvaluation = eval; }
public void setEvaluation(Evaluation eval) { evaluation = eval; }
public String toJson() { return Json.writeValueAsStringHard(this); }
public static Example fromJson(String json) { return Json.readValueHard(json, Example.class); }
public Evaluation computeTotalEvaluation() {
Evaluation eval = new Evaluation();
if (parseEvaluation != null)
eval.add(parseEvaluation);
if (evaluation != null)
eval.add(evaluation);
return eval;
}
void rescoreAndSortPredDerivations(Params params) {
for (Derivation deriv : predDerivations)
deriv.computeScore(params);
Derivation.sortByScore(predDerivations);
}
public String toJson() {
return Json.writeValueAsStringHard(this);
}
public static Example fromJson(String json) {
return Json.readValueHard(json, Example.class);
}
/** Use JSON instead. */
@Deprecated
public static Example fromLispTree(LispTree tree) {
return fromLispTree(tree, null);
}
@Deprecated
public static Example fromLispTree(LispTree tree, String defaultId) {
Builder b = new Builder().setId(defaultId);
@ -225,8 +139,11 @@ public class Example {
if (arg.children.size() != 2)
throw new RuntimeException("Expect one target value");
b.setTargetValue(Values.fromLispTree(arg.child(1)));
} else if ("context".equals(label)) {
b.setContext(new ContextValue(arg));
}
}
b.setLanguageInfo(new LanguageInfo());
Example ex = b.createExample();
@ -237,18 +154,15 @@ public class Example {
// Do nothing
} else if ("tokens".equals(label)) {
int n = arg.child(1).children.size();
ex.backupTokens = new ArrayList<String>(n);
for (int j = 0; j < n; j++)
ex.backupTokens.add(arg.child(1).child(j).value);
ex.languageInfo.tokens.add(arg.child(1).child(j).value);
} else if ("evaluation".equals(label)) {
ex.evaluation = Evaluation.fromLispTree(arg.child(1));
} else if ("parseEvaluation".equals(label)) {
ex.parseEvaluation = Evaluation.fromLispTree(arg.child(1));
} else if ("predDerivations".equals(label)) {
ex.predDerivations = new ArrayList<Derivation>();
ex.predDerivations = new ArrayList<>();
for (int j = 1; j < arg.children.size(); j++)
ex.predDerivations.add(derivationFromLispTree(arg.child(j)));
} else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues").contains(label)) {
} else if (!Sets.newHashSet("id", "utterance", "targetFormula", "targetValue", "targetValues", "context").contains(label)) {
throw new RuntimeException("Invalid example argument: " + arg);
}
}
@ -256,11 +170,8 @@ public class Example {
return ex;
}
public void preprocess() {
if (this.languageInfo == null)
this.languageInfo = new LanguageInfo();
this.languageInfo.analyze(this.utterance);
this.log();
public void preprocess(LanguageAnalyzer analyzer) {
this.languageInfo = analyzer.analyze(this.utterance);
}
public void log() {
@ -270,20 +181,32 @@ public class Example {
LogInfo.logs("POS tags: %s", languageInfo.posTags);
LogInfo.logs("NER tags: %s", languageInfo.nerTags);
LogInfo.logs("NER values: %s", languageInfo.nerValues);
if (context != null)
LogInfo.logs("context: %s", context);
if (targetFormula != null)
LogInfo.logs("targetFormula: %s", targetFormula);
if (targetValue != null)
LogInfo.logs("targetValue: %s", targetValue);
LogInfo.logs("Dependency children: %s", languageInfo.dependencyChildren);
LogInfo.end_track();
}
// To save memory
public void clearPredDerivations() {
predDerivations.clear();
predDerivationsAfterParse.clear();
}
/** Use JSON serialization instead. */
@Deprecated
public List<Derivation> getCorrectDerivations() {
List<Derivation> res = new ArrayList<>();
for (Derivation deriv : predDerivations) {
if (deriv.compatibility == Double.NaN)
throw new RuntimeException("Compatibility is not set");
if (deriv.compatibility > 0)
res.add(deriv);
}
return res;
}
public LispTree toLispTree(boolean outputPredDerivations) {
LispTree tree = LispTree.proto.newList();
tree.addChild("example");
@ -308,8 +231,6 @@ public class Example {
if (evaluation != null)
tree.addChild(LispTree.proto.newList("evaluation", evaluation.toLispTree()));
if (parseEvaluation != null)
tree.addChild(LispTree.proto.newList("parseEvaluation", parseEvaluation.toLispTree()));
if (predDerivations != null && outputPredDerivations) {
LispTree list = LispTree.proto.newList();
@ -322,7 +243,6 @@ public class Example {
return tree;
}
@Deprecated
private static Derivation derivationFromLispTree(LispTree item) {
Derivation.Builder b = new Derivation.Builder()
.cat(Rule.rootCat)
@ -352,12 +272,9 @@ public class Example {
return b.createDerivation();
}
@Deprecated
private static LispTree derivationToLispTree(Derivation deriv) {
LispTree item = LispTree.proto.newList();
// TODO: label scores and compatibilities in derivations to make output
// more self-documenting.
item.addChild(deriv.compatibility + "");
item.addChild(deriv.prob + "");
item.addChild(deriv.score + "");
@ -367,7 +284,7 @@ public class Example {
item.addChild("null");
item.addChild(deriv.formula.toLispTree());
HashMap<String, Double> features = new HashMap<String, Double>();
HashMap<String, Double> features = new HashMap<>();
deriv.incrementAllFeatureVector(1, features);
item.addChild(LispTree.proto.newList(features));

63
src/edu/stanford/nlp/sempre/ExampleUtils.java Normal file
View File

@ -0,0 +1,63 @@
package edu.stanford.nlp.sempre;
import fig.basic.*;
import fig.exec.Execution;
import java.io.*;
import java.util.*;
/**
* Output examples in various forms.
*
* @author Percy Liang
*/
public final class ExampleUtils {
private ExampleUtils() { }
// Output JSON file with just the basic input/output.
public static void writeJson(List<Example> examples) {
PrintWriter out = IOUtils.openOutHard(Execution.getFile("examples.json"));
for (Example ex : examples)
out.println(ex.toJson());
out.close();
}
private static String escapeSpace(String s) {
return s.replaceAll(" ", "&nbsp;");
}
// Output examples in Simple Dataset Format (Ranking).
public static void writeSDF(int iter, String group,
Evaluation evaluation,
List<Example> examples,
boolean outputPredDerivations) {
String basePath = "preds-iter" + iter + "-" + group + ".examples";
String outPath = Execution.getFile(basePath);
if (outPath == null || examples.size() == 0) return;
LogInfo.begin_track("Writing examples to %s", basePath);
PrintWriter out = IOUtils.openOutHard(outPath);
LispTree p = LispTree.proto;
out.println("# " + p.L(p.L("iter", iter), p.L("group", group), p.L("numExamples", examples.size()), p.L("evaluation", evaluation.toLispTree())));
for (Example ex : examples) {
out.println("");
out.println("example " + ex.id);
out.println("description " + p.L(p.L("utterance", ex.utterance), p.L("targetValue", ex.targetValue.toLispTree()), p.L("evaluation", ex.evaluation.toLispTree())));
if (outputPredDerivations) {
for (Derivation deriv : ex.predDerivations) {
StringBuilder buf = new StringBuilder();
buf.append("item " + escapeSpace(p.L(p.L("formula", deriv.formula.toLispTree()), p.L("value", deriv.value.toLispTree())).toString())); // Description
buf.append(" " + deriv.compatibility);
Map<String, Double> features = deriv.getAllFeatureVector();
for (Map.Entry<String, Double> e : features.entrySet()) {
buf.append(" " + escapeSpace(e.getKey()) + ":" + e.getValue());
}
out.println(buf.toString());
}
}
}
out.close();
LogInfo.end_track();
}
}

6
src/edu/stanford/nlp/sempre/Executor.java
View File

@ -1,5 +1,7 @@
package edu.stanford.nlp.sempre;
import fig.basic.Evaluation;
/**
* An Executor takes a logical form (Formula) and computes its denotation
* (Value).
@ -17,6 +19,6 @@ public abstract class Executor {
public final Evaluation stats;
}
// Execute the formula on the database.
public abstract Response execute(Formula formula);
// Execute the formula in the given context.
public abstract Response execute(Formula formula, ContextValue context);
}

25
src/edu/stanford/nlp/sempre/FeatureComputer.java Normal file
View File

@ -0,0 +1,25 @@
package edu.stanford.nlp.sempre;
/**
* A feature computer.
*
* Look at a derivation and add features to the feature vector.
* A FeatureComputer should be stateless.
*
* Before computing features, a FeatureComputer should call
*
* if (!FeatureExtractor.containsDomain(...)) return;
*
* to check the feature domain first.
*/
public interface FeatureComputer {
/**
* This function is called on every sub-Derivation.
*
* It should extract only the features which depend in some way on |deriv|,
* not just on its children.
*/
void extractLocal(Example ex, Derivation deriv);
}

309
src/edu/stanford/nlp/sempre/FeatureExtractor.java
View File

@ -1,19 +1,23 @@
package edu.stanford.nlp.sempre;
import java.util.*;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import fig.basic.Option;
import fig.basic.StopWatchSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import fig.basic.*;
/**
* A FeatureExtractor specifies a mapping from derivations to feature vectors.
* There are two ways to add features to a Derivation:
* 1) Add FeatureExtractor classes, which are called on each sub-Derivation.
* 2) Use SemanticFn, which are called whenever they are called.
*
* [Using features]
* - Specify the feature domains in the featureDomains option.
* - If the features are defined in a separate FeatureComputer class, also specify
* the class name in the featureComputers option.
*
* [Implementing new features] There are 3 ways to implement new features:
* 1) Define features in SemanticFn, which is called when the SemanticFn is called.
* 2) Create a FeatureComputer class, which is called on each sub-Derivation if the class name
* is specified in the featureComputers option.
* 3) Add a method to this class. The method is called on each sub-Derivation.
*
* @author Percy Liang
*/
@ -21,22 +25,25 @@ public class FeatureExtractor {
// Global place to specify features.
public static class Options {
@Option(gloss = "Set of feature domains to include")
public Set<String> featureDomains = Sets.newHashSet();
public Set<String> featureDomains = new HashSet<>();
@Option(gloss = "Set of feature computer classes to load")
public Set<String> featureComputers = Sets.newHashSet("DerivOpCountFeatureComputer");
@Option(gloss = "Disable denotation features")
public boolean disableDenotationFeatures = false;
@Option(gloss = "Use all possible features, regardless of what featureDomains says")
public boolean useAllFeatures = false;
@Option(gloss = "Whether to conjoin all lemmas with binaries or each lemma")
public boolean conjoinAllLemmas = false;
@Option(gloss = "For bigram features in paraphrased utterances, maximum distance to consider")
public int maxBigramDistance = 3;
}
private Executor executor;
private DerivOpCountFeatureExtractor opCountExtractor = new DerivOpCountFeatureExtractor();
private List<FeatureComputer> featureComputers = new ArrayList<>();
public FeatureExtractor(Executor executor) {
this.executor = executor;
for (String featureComputer : opts.featureComputers) {
featureComputers.add((FeatureComputer) Utils.newInstanceHard(SempreUtils.resolveClassName(featureComputer)));
}
}
public static Options opts = new Options();
@ -51,14 +58,19 @@ public class FeatureExtractor {
public void extractLocal(Example ex, Derivation deriv) {
StopWatchSet.begin("FeatureExtractor.extractLocal");
extractRuleFeatures(ex, deriv);
extractSpanFeatures(ex, deriv);
extractDenotationFeatures(ex, deriv);
extractDependencyFeatures(ex, deriv);
extractWhTypeFeatures(ex, deriv);
opCountExtractor.extractLocal(ex, deriv);
conjoinLemmaAndBinary(ex, deriv);
extractParaphraseFeatures(ex, deriv);
extractBigramFeatures(ex, deriv);
for (FeatureComputer featureComputer : featureComputers)
featureComputer.extractLocal(ex, deriv);
StopWatchSet.end();
}
// Add an indicator for each rule that's applied.
// Add an indicator for each applied rule.
void extractRuleFeatures(Example ex, Derivation deriv) {
if (!containsDomain("rule")) return;
if (deriv.rule != Rule.nullRule) {
@ -67,38 +79,96 @@ public class FeatureExtractor {
}
}
// Extract features on the linguistic information of the spanned (anchored) tokens.
// (Not applicable for floating rules)
void extractSpanFeatures(Example ex, Derivation deriv) {
if (!containsDomain("span") || deriv.start == -1) return;
deriv.addFeature("span", "cat=" + deriv.cat + ",#tokens=" + (deriv.end - deriv.start));
deriv.addFeature("span", "cat=" + deriv.cat + ",POS=" + ex.posTag(deriv.start) + "..." + ex.posTag(deriv.end - 1));
}
// Extract features on the denotation of the logical form produced.
// (For example, number of items in the list)
void extractDenotationFeatures(Example ex, Derivation deriv) {
if (!containsDomain("denotation")) return;
if (!deriv.isRoot(ex.numTokens())) return;
deriv.ensureExecuted(executor);
deriv.ensureExecuted(executor, ex.context);
if (deriv.value instanceof ErrorValue) {
deriv.addFeature("denotation", "error");
return;
}
if (!(deriv.value instanceof ListValue))
throw new RuntimeException("Derivation value is not a list: " + deriv.value);
if (deriv.value instanceof ListValue) {
ListValue list = (ListValue) deriv.value;
ListValue list = (ListValue) deriv.value;
if (Formulas.isCountFormula(deriv.formula)) {
// TODO: clean this up
if (list.values.size() != 1) {
throw new RuntimeException(
"Evaluation of count formula " + deriv.formula + " has size " + list.values.size());
// TODO(pliang): this is hacky; don't depend on the logical form
if (Formulas.isCountFormula(deriv.formula)) {
if (list.values.size() != 1) {
deriv.addFeature("denotation", "size", list.values.size());
} else {
int count = getNumber(list.values.get(0));
deriv.addFeature("denotation", "count-size" + (count == 0 ? "=0" : ">0"));
}
} else {
int size = list.values.size();
deriv.addFeature("denotation", "size" + (size < 3 ? "=" + size : ">=" + 3));
}
int count = (int)((NumberValue)list.values.get(0)).value;
deriv.addFeature("denotation", "count-size" + (count == 0 ? "=0" : ">0"));
} else {
int size = list.values.size();
deriv.addFeature("denotation", "size" + (size < 3 ? "=" + size : ">=" + 3));
}
}
// Conjunction of wh and type
int getNumber(Value value) {
if (value instanceof NumberValue) return (int) ((NumberValue) value).value;
if (value instanceof ListValue) return getNumber(((ListValue) value).values.get(0));
throw new RuntimeException("Can't extract number from " + value);
}
// Add an indicator for each alignment between a syntactic dependency (produced by the
// Stanford dependency parser) and the application of a semantic function.
void extractDependencyFeatures(Example ex, Derivation deriv) {
if (!containsDomain("dependencyParse") && !containsDomain("fullDependencyParse")) return;
if (deriv.rule != Rule.nullRule) {
for (Derivation child : deriv.children) {
for (int i = child.start; i < child.end; i++) {
for (LanguageInfo.DependencyEdge dependency : ex.languageInfo.dependencyChildren.get(i)) {
if (!child.containsIndex(dependency.modifier)) {
String direction = dependency.modifier > i ? "forward" : "backward";
String containment = deriv.containsIndex(dependency.modifier) ? "internal" : "external";
if (containsDomain("fullDependencyParse"))
addAllDependencyFeatures(dependency, direction, containment,
deriv);
else
deriv.addFeature("dependencyParse",
"(" + dependency.label + " " + direction + " " + containment + ") --- "
+ deriv.getRule().toString());
}
}
}
}
}
}
private void addAllDependencyFeatures(LanguageInfo.DependencyEdge dependency,
String direction, String containment, Derivation deriv) {
String[] types = {dependency.label, "*"};
String[] directions = {" " + direction, ""};
String[] containments = {" " + containment, ""};
String[] rules = {deriv.getRule().toString(), ""};
for (String typePresent : types) {
for (String directionPresent : directions) {
for (String containmentPresent : containments) {
for (String rulePresent : rules) {
deriv.addFeature("fullDependencyParse",
"(" + typePresent + directionPresent + containmentPresent + ") --- " + rulePresent);
}
}
}
}
}
// Conjunction of wh-question word and type
// (For example, "who" should go with PERSON and not DATE)
void extractWhTypeFeatures(Example ex, Derivation deriv) {
if (!containsDomain("whType")) return;
if (!deriv.isRoot(ex.numTokens())) return;
@ -106,61 +176,172 @@ public class FeatureExtractor {
if (ex.posTag(0).startsWith("W")) {
deriv.addFeature("whType",
"token0=" + ex.token(0) + "," +
"type=" + FreebaseInfo.getSingleton().coarseType(deriv.type.toString()));
"type=" + coarseType(deriv.type.toString()));
}
}
public static final String PERSON = "fb:people.person";
public static final String LOC = "fb:location.location";
public static final String ORG = "fb:organization.organization";
public static String coarseType(String type) {
Set<String> superTypes = SemTypeHierarchy.singleton.getSupertypes(type);
if (superTypes != null) {
if (superTypes.contains(PERSON)) return PERSON;
if (superTypes.contains(LOC)) return LOC;
if (superTypes.contains(ORG)) return ORG;
if (superTypes.contains(CanonicalNames.NUMBER)) return CanonicalNames.NUMBER;
if (superTypes.contains(CanonicalNames.DATE)) return CanonicalNames.DATE;
}
return "OTHER";
}
//used in Berant et al., 2013 and in the RL parser
//conjoins all binaries in the logical form with all non-entity lemmas
void conjoinLemmaAndBinary(Example ex, Derivation deriv) {
if(!containsDomain("lemmaAndBinaries")) return;
if (!containsDomain("lemmaAndBinaries")) return;
if (!deriv.isRoot(ex.numTokens())) return;
List<String> nonEntityLemmas = new LinkedList<String>();
extractNonEntityLemmas(ex,deriv,nonEntityLemmas);
List<String> nonEntityLemmas = new LinkedList<>();
extractNonEntityLemmas(ex, deriv, nonEntityLemmas);
List<String> binaries = extractBinaries(deriv.formula);
if(opts.conjoinAllLemmas) {
deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas="+Joiner.on('_').join(nonEntityLemmas)+
",binaries="+Joiner.on('_').join(binaries));
}
else {
String binariesStr = Joiner.on('_').join(binaries);
for(String nonEntityLemma: nonEntityLemmas) {
deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas="+nonEntityLemma+
",binaries="+binariesStr);
}
String binariesStr = Joiner.on('_').join(binaries);
for (String nonEntityLemma : nonEntityLemmas) {
deriv.addFeature("lemmaAndBinaries", "nonEntitylemmas=" + nonEntityLemma +
",binaries=" + binariesStr);
}
}
//TODO clean this
// Extract the utterance that the derivation generates (not necessarily the
// one in the input utterance).
private void extractUtterance(Derivation deriv, List<String> utterance) {
if (deriv.rule == Rule.nullRule) return;
int c = 0; // Index into children
for (String item : deriv.rule.rhs) {
if (Rule.isCat(item))
extractUtterance(deriv.children.get(c++), utterance);
else
utterance.add(item);
}
}
//Used in Berant et., EMNLP 2013, and in the agenda RL parser
//Extracts all content-word lemmas in the derivation tree not dominated by the category $Entity
private void extractNonEntityLemmas(Example ex, Derivation deriv,
List<String> nonEntityLemmas) {
if(deriv.children.size()==0) {//base case this means it is a word that should be appended
for(int i = deriv.start; i < deriv.end; i++){
if (deriv.children.size() == 0) { // base case this means it is a word that should be appended
for (int i = deriv.start; i < deriv.end; i++) {
String pos = ex.languageInfo.posTags.get(i);
if((pos.startsWith("N") || pos.startsWith("V") || pos.startsWith("W") || pos.startsWith("A") || pos.equals("IN"))
&& !ex.languageInfo.lemmaTokens.get(i).equals("be"))
if ((pos.startsWith("N") || pos.startsWith("V") || pos.startsWith("W") || pos.startsWith("A") || pos.equals("IN"))
&& !ex.languageInfo.lemmaTokens.get(i).equals("be"))
nonEntityLemmas.add(ex.languageInfo.lemmaTokens.get(i));
}
}
else { //recursion
for(Derivation child: deriv.children) {
if(child.rule.lhs == null || !child.rule.lhs.equals("$Entity")) {
} else { // recursion
for (Derivation child : deriv.children) {
if (child.rule.lhs == null || !child.rule.lhs.equals("$Entity")) {
extractNonEntityLemmas(ex, child, nonEntityLemmas);
}
else if(child.rule.lhs.equals("$Entity")) {
} else if (child.rule.lhs.equals("$Entity")) {
nonEntityLemmas.add("E");
}
}
}
}
//TODO clean this
//Used in Berant et al., 2013 and in agenda-based RL parser
private List<String> extractBinaries(Formula formula) {
List<String> res = new LinkedList<String>();
List<String> res = new LinkedList<>();
Set<String> atomicElements = Formulas.extractAtomicFreebaseElements(formula);
for(String atomicElement: atomicElements) {
if(atomicElement.split("\\.").length==3 && !atomicElement.equals("fb:type.object.type"))
for (String atomicElement : atomicElements) {
if (atomicElement.split("\\.").length == 3 && !atomicElement.equals("fb:type.object.type"))
res.add(atomicElement);
}
return res;
}
/**
* Add an indicator for each token alignment between a token in the utterance and its
* canonical form in the grammar using the ParaphraseModel, if applicable.
*/
// Compute paraphrasing features. In the future, need to store temporary
// state to make this more efficient.
void extractParaphraseFeatures(Example ex, Derivation deriv) {
if (!containsDomain("paraphrasing")) return;
if (deriv.rule == Rule.nullRule) return;
extractParaphrasePrecision(ex, deriv);
extractParaphraseRecall(ex, deriv);
// Make sure we have a valid, floating, paraphraseable rule
if (!deriv.rule.isFloating() || !deriv.rule.isRhsTerminals()) return;
String key = join(deriv.rule.rhs, " ");
ParaphraseModel model = ParaphraseModel.getSingleton();
if (!model.containsKey(key)) return;
for (String token : ex.getTokens()) {
double score = model.get(key, token);
if (score > 0.0) {
deriv.addFeature("paraphrasing", "(" + key + ", " + token + ")");
}
}
}
/**
* Add an indicator for each pair of bigrams that can be aligned from the original
* utterance and two (not necessarily contiguous) lemmas in the generated utterance
*/
private void extractBigramFeatures(Example ex, Derivation deriv) {
if (!containsDomain("bigram")) return;
if (!deriv.cat.equals(Rule.rootCat)) return;
LanguageInfo derivInfo = LanguageAnalyzer.getSingleton().analyze(deriv.canonicalUtterance);
List<String> derivLemmas = derivInfo.lemmaTokens;
List<String> exLemmas = ex.languageInfo.lemmaTokens;
for (int i = 0; i < exLemmas.size() - 1; i++) {
for (int j = 0; j < derivLemmas.size() - 1; j++) {
if (derivLemmas.get(j).equals(exLemmas.get(i))) {
// Consider bigrams separated by up to maxBigramDistance in generated utterance
for (int k = 1; j + k < derivLemmas.size() && k <= opts.maxBigramDistance; k++) {
if (derivLemmas.get(j + k).equals(exLemmas.get(i + 1))) {
deriv.addFeature("bigram",
exLemmas.get(i) + "," + exLemmas.get(i + 1) + " - " + k);
}
}
}
}
}
}
private void extractParaphraseRecall(Example ex, Derivation deriv) {
if (!deriv.cat.equals(Rule.rootCat)) return;
String[] derivTokens = deriv.canonicalUtterance.split("\\s+");
int n = 0;
for (int i = 0; i < ex.languageInfo.numTokens(); ++i) {
for (int j = 0; j < derivTokens.length; ++j) {
if (derivTokens[j].equals(ex.languageInfo.tokens.get(i))) {
n++;
break;
}
}
}
deriv.addFeature("paraphrasing", "deriv_recall", (double) n / ex.languageInfo.numTokens());
}
private void extractParaphrasePrecision(Example ex, Derivation deriv) {
//how many of the generated stuff is in the original utterance
int n = 0;
for (String item : deriv.rule.rhs) {
if (Rule.isCat(item)) continue;
if (ex.languageInfo.tokens.contains(item))
n++;
}
deriv.addFeature("paraphrasing", "deriv_prec", n);
}
// Joins arrayList of strings into string
String join(List<String> l, String delimiter) {
StringBuilder sb = new StringBuilder(l.get(0));
for (int i = 1; i < l.size(); i++) {
sb.append(delimiter);
sb.append(l.get(i));
}
return sb.toString();
}
}

8
src/edu/stanford/nlp/sempre/FeatureMatcher.java
View File

@ -1,24 +1,24 @@
package edu.stanford.nlp.sempre;
public interface FeatureMatcher {
public boolean matches(String feature);
boolean matches(String feature);
}
class AllFeatureMatcher implements FeatureMatcher {
final class AllFeatureMatcher implements FeatureMatcher {
private AllFeatureMatcher() { }
@Override
public boolean matches(String feature) { return true; }
public static final AllFeatureMatcher matcher = new AllFeatureMatcher();
}
class ExactFeatureMatcher implements FeatureMatcher {
final class ExactFeatureMatcher implements FeatureMatcher {
private String match;
public ExactFeatureMatcher(String match) { this.match = match; }
@Override
public boolean matches(String feature) { return feature.equals(match); }
}
class DenotationFeatureMatcher implements FeatureMatcher {
final class DenotationFeatureMatcher implements FeatureMatcher {
@Override
public boolean matches(String feature) {
return feature.startsWith("denotation-size") ||

107
src/edu/stanford/nlp/sempre/FeatureVector.java
View File

@ -14,21 +14,22 @@ import java.util.*;
* so that the key space isn't a free-for-all.
*
* @author Percy Liang
* @author Jonathan Berant
*/
public class FeatureVector {
// These features map to the value 1 (most common case in NLP).
private ArrayList<String> indicatorFeatures;
// General features
private ArrayList<Pair<String, Double>> generalFeatures;
//A dense array of features to save memory
// A dense array of features to save memory
private double[] denseFeatures;
private static final String DENSE_NAME = "Dns";
public FeatureVector() {} //constructor that does nothing
public FeatureVector() { } // constructor that does nothing
public FeatureVector(int numOfDenseFeatures) {
denseFeatures=new double[numOfDenseFeatures];
Arrays.fill(denseFeatures,0d);
denseFeatures = new double[numOfDenseFeatures];
Arrays.fill(denseFeatures, 0d);
}
private static String toFeature(String domain, String name) { return domain + " :: " + name; }
@ -37,7 +38,7 @@ public class FeatureVector {
add(toFeature(domain, name));
}
private void add(String feature) {
if (indicatorFeatures == null) indicatorFeatures = new ArrayList<String>();
if (indicatorFeatures == null) indicatorFeatures = new ArrayList<>();
indicatorFeatures.add(feature);
}
@ -45,7 +46,7 @@ public class FeatureVector {
add(toFeature(domain, name), value);
}
private void add(String feature, double value) {
if (generalFeatures == null) generalFeatures = new ArrayList<Pair<String, Double>>();
if (generalFeatures == null) generalFeatures = new ArrayList<>();
generalFeatures.add(Pair.newPair(feature, value));
}
@ -54,6 +55,30 @@ public class FeatureVector {
add(domain, name + "-bias", 1);
}
// Add histogram features, e.g., domain :: name>=4
public void addHistogram(String domain, String name, double value) { addHistogram(domain, name, value, 2, 10, true); }
public void addHistogram(String domain, String name, double value, int initBinSize, int numBins, boolean exp) {
double upper = initBinSize;
String bin = null;
int sign = value > 0 ? +1 : -1;
value = Math.abs(value);
for (int i = 0; i < numBins; i++) {
double lastUpper = upper;
if (i > 0) {
if (exp) upper *= initBinSize;
else upper += initBinSize;
}
if (value < upper) {
bin = (sign > 0) ? lastUpper + ":" + upper : (-upper) + ":" + (-lastUpper);
break;
}
}
if (bin == null)
bin = (sign > 0) ? ">=" + upper : "<=" + (-upper);
add(domain, name + bin);
}
public void addFromString(String feature, double value) {
assert feature.contains(" :: ") : feature;
if (value == 1) add(feature);
@ -61,7 +86,7 @@ public class FeatureVector {
}
public void addDenseFeature(int index, double value) {
denseFeatures[index]+=value;
denseFeatures[index] += value;
}
public void add(FeatureVector that) { add(that, AllFeatureMatcher.matcher); }
@ -76,11 +101,11 @@ public class FeatureVector {
if (matcher.matches(pair.getFirst()))
add(pair.getFirst(), pair.getSecond());
}
//dense features are always added
if(that.denseFeatures!=null) {
for(int i = 0; i < denseFeatures.length;++i)
denseFeatures[i]=that.denseFeatures[i];
}
// dense features are always added
if (that.denseFeatures != null) {
for (int i = 0; i < denseFeatures.length; ++i)
denseFeatures[i] = that.denseFeatures[i];
}
}
// Return the dot product between this feature vector and the weight vector (parameters).
@ -94,9 +119,9 @@ public class FeatureVector {
for (Pair<String, Double> pair : generalFeatures)
sum += params.getWeight(pair.getFirst()) * pair.getSecond();
}
if(denseFeatures != null) {
for(int i = 0; i < denseFeatures.length; ++i)
sum += params.getWeight(DENSE_NAME+"_"+i) * denseFeatures[i];
if (denseFeatures != null) {
for (int i = 0; i < denseFeatures.length; ++i)
sum += params.getWeight(DENSE_NAME + "_" + i) * denseFeatures[i];
}
return sum;
}
@ -118,18 +143,33 @@ public class FeatureVector {
MapUtils.incr(map, pair.getFirst(), factor * pair.getSecond());
}
if (denseFeatures != null) {
for(int i = 0; i < denseFeatures.length;++i)
MapUtils.incr(map, DENSE_NAME+"_"+i, factor * denseFeatures[i]);
for (int i = 0; i < denseFeatures.length; ++i)
MapUtils.incr(map, DENSE_NAME + "_" + i, factor * denseFeatures[i]);
}
}
// returns a feature vector where all features are prefixed
public FeatureVector addPrefix(String prefix) {
FeatureVector res = new FeatureVector();
if (indicatorFeatures != null) {
for (String feature : indicatorFeatures)
res.add(prefix + feature);
}
if (generalFeatures != null) {
for (Pair<String, Double> pair : generalFeatures) {
res.add(prefix + pair.getFirst(), pair.getSecond());
}
}
return res;
}
@JsonValue
public Map<String, Double> toMap() {
HashMap<String, Double> map = new HashMap<String, Double>();
increment(1, map);
if(denseFeatures!=null) {
for(int i = 0; i < denseFeatures.length; ++i) {
map.put(DENSE_NAME+"_"+i,denseFeatures[i]);
if (denseFeatures != null) {
for (int i = 0; i < denseFeatures.length; ++i) {
map.put(DENSE_NAME + "_" + i, denseFeatures[i]);
}
}
return map;
@ -140,22 +180,21 @@ public class FeatureVector {
// TODO (rf):
// Encoding is lossy. We guess that value of 1 means indicator, but we
// could be wrong.
//(joberant) - takes care of dense features in a non efficient way - TODO
// TODO(joberant) - takes care of dense features in a non efficient way
int maxDenseFeaturesIndex = -1;
for (Map.Entry<String, Double> entry : m.entrySet()) {
if(isDenseFeature(entry.getKey())) {
if (isDenseFeature(entry.getKey())) {
int index = denseFeatureIndex(entry.getKey());
if(index>maxDenseFeaturesIndex)
maxDenseFeaturesIndex=index;
if (index > maxDenseFeaturesIndex)
maxDenseFeaturesIndex = index;
}
}
FeatureVector fv = maxDenseFeaturesIndex==-1 ? new FeatureVector() : new FeatureVector(maxDenseFeaturesIndex+1);
FeatureVector fv = maxDenseFeaturesIndex == -1 ? new FeatureVector() : new FeatureVector(maxDenseFeaturesIndex + 1);
for (Map.Entry<String, Double> entry : m.entrySet()) {
if(isDenseFeature(entry.getKey())) {
if (isDenseFeature(entry.getKey())) {
fv.addDenseFeature(denseFeatureIndex(entry.getKey()), entry.getValue());
}
else {
} else {
if (entry.getValue() == 1.0d)
fv.add(entry.getKey());
else
@ -203,18 +242,18 @@ public class FeatureVector {
}
LogInfo.end_track();
}
public static void logFeatures(Map<String, Double> features) {
for(String key: features.keySet()) {
LogInfo.logs("%s\t%s",key,features.get(key));
for (String key : features.keySet()) {
LogInfo.logs("%s\t%s", key, features.get(key));
}
}
public void clear() {
if(indicatorFeatures!=null)
if (indicatorFeatures != null)
indicatorFeatures.clear();
if(generalFeatures!=null)
if (generalFeatures != null)
generalFeatures.clear();
denseFeatures=null;
denseFeatures = null;
}
}

43
src/edu/stanford/nlp/sempre/FilterNerSpanFn.java
View File

@ -3,7 +3,6 @@ package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
@ -14,7 +13,7 @@ import java.util.List;
*/
public class FilterNerSpanFn extends SemanticFn {
// Accepted NER tags (PERSON, LOCATION, ORGANIZATION, etc)
List<String> acceptableNerTags = new ArrayList<String>();
List<String> acceptableNerTags = new ArrayList<>();
public void init(LispTree tree) {
super.init(tree);
@ -22,40 +21,42 @@ public class FilterNerSpanFn extends SemanticFn {
acceptableNerTags.add(tree.child(j).value);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FilterNerSpanFn that = (FilterNerSpanFn) o;
if (!acceptableNerTags.equals(that.acceptableNerTags)) return false;
return true;
public DerivationStream call(final Example ex, final Callable c) {
return new SingleDerivationStream() {
@Override
public Derivation createDerivation() {
if (!isValid(ex, c))
return null;
else {
return new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation();
}
}
};
}
public List<Derivation> call(Example ex, Callable c) {
private boolean isValid(Example ex, Callable c) {
String nerTag = ex.languageInfo.nerTags.get(c.getStart());
// Check that it's an acceptable tag
if (!acceptableNerTags.contains(nerTag))
return Collections.emptyList();
return false;
// Check to make sure that all the tags are the same
for (int j = c.getStart() + 1; j < c.getEnd(); j++)
if (!nerTag.equals(ex.languageInfo.nerTags.get(j)))
return Collections.emptyList();
return false;
// Make sure that the whole NE is matched
if (c.getStart() > 0 && nerTag.equals(ex.languageInfo.nerTags.get(c.getStart() - 1)))
return Collections.emptyList();
return false;
if (c.getEnd() < ex.languageInfo.nerTags.size() &&
nerTag.equals(ex.languageInfo.nerTags.get(c.getEnd())))
return Collections.emptyList();
nerTag.equals(ex.languageInfo.nerTags.get(c.getEnd())))
return false;
assert (c.getChildren().size() == 1) : c.getChildren();
return Collections.singletonList(
new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation());
return true;
}
}

88
src/edu/stanford/nlp/sempre/FilterPosTagFn.java
View File

@ -3,7 +3,6 @@ package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
@ -14,8 +13,9 @@ import java.util.List;
*/
public class FilterPosTagFn extends SemanticFn {
// Accepted POS tags (e.g., NNP, NNS, etc.)
List<String> posTags = new ArrayList<String>();
List<String> posTags = new ArrayList<>();
String mode;
boolean reverse;
public void init(LispTree tree) {
super.init(tree);
@ -23,62 +23,76 @@ public class FilterPosTagFn extends SemanticFn {
if (!mode.equals("span") && !mode.equals("token"))
throw new RuntimeException("Illegal description for whether to filter by token or span: " + tree.child(1).value);
for (int j = 2; j < tree.children.size(); j++)
for (int j = 2; j < tree.children.size(); j++) {
// Optionally, we can use a reverse filter (only reject certain tags)
if (j == 2 && tree.child(2).value.equals("reverse")) {
reverse = true;
continue;
}
posTags.add(tree.child(j).value);
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FilterPosTagFn that = (FilterPosTagFn) o;
if (!mode.equals(that.mode)) return false;
if (!posTags.equals(that.posTags)) return false;
return true;
public DerivationStream call(final Example ex, final Callable c) {
return new SingleDerivationStream() {
@Override
public Derivation createDerivation() {
if (mode.equals("span"))
return callSpan(ex, c);
else
return callToken(ex, c);
}
};
}
public List<Derivation> call(Example ex, Callable c) {
if (mode.equals("span"))
return callSpan(ex, c);
return callToken(ex, c);
}
private List<Derivation> callToken(Example ex, Callable c) {
private Derivation callToken(Example ex, Callable c) {
// Only apply to single tokens
if (c.getEnd() - c.getStart() != 1)
return Collections.emptyList();
String posTag = ex.posTag(c.getStart());
if (posTags.contains(posTag)) {
return Collections.singletonList(
new Derivation.Builder()
if (c.getEnd() - c.getStart() != 1 ||
(!posTags.contains(posTag) ^ reverse))
return null;
else {
return new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation());
.createDerivation();
}
return Collections.emptyList();
}
private List<Derivation> callSpan(Example ex, Callable c) {
private Derivation callSpan(Example ex, Callable c) {
if (isValidSpan(ex, c)) {
return new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation();
} else {
return null;
}
}
private boolean isValidSpan(Example ex, Callable c) {
if (reverse) {
for (int j = c.getStart(); j < c.getEnd(); j++) {
if (posTags.contains(ex.posTag(j)))
return false;
}
return true;
}
String posTag = ex.posTag(c.getStart());
// Check that it's an acceptable tag
if (!posTags.contains(posTag)) return Collections.emptyList();
if (!posTags.contains(posTag))
return false;
// Check to make sure that all the tags are the same
for (int j = c.getStart() + 1; j < c.getEnd(); j++) {
if (!posTag.equals(ex.posTag(j)))
return Collections.emptyList();
return false;
}
// Make sure that the whole POS sequence is matched
if (c.getStart() > 0 && posTag.equals(ex.posTag(c.getStart() - 1)))
return Collections.emptyList();
return false;
if (c.getEnd() < ex.numTokens() && posTag.equals(ex.posTag(c.getEnd())))
return Collections.emptyList();
return false;
assert (c.getChildren().size() == 1) : c.getChildren();
return Collections.singletonList(
new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation());
return true;
}
}

48
src/edu/stanford/nlp/sempre/FilterSpanLengthFn.java
View File

@ -2,36 +2,42 @@ package edu.stanford.nlp.sempre;
import fig.basic.LispTree;
import java.util.Collections;
import java.util.List;
public class FilterSpanLengthFn extends SemanticFn {
private int minLength;
private int maxLength;
private static final int NO_MAXIMUM = -1;
public FilterSpanLengthFn() { }
public FilterSpanLengthFn(int minLength) {
init(LispTree.proto.newList("FilterSpanLengthFn", "" + minLength));
}
public void init(LispTree tree) {
super.init(tree);
minLength = Integer.parseInt(tree.child(1).value);
if (tree.children.size() > 2) {
maxLength = Integer.parseInt(tree.child(2).value);
} else {
maxLength = NO_MAXIMUM;
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FilterSpanLengthFn that = (FilterSpanLengthFn) o;
if (minLength != that.minLength) return false;
return true;
public DerivationStream call(Example ex, final Callable c) {
return new SingleDerivationStream() {
@Override
public Derivation createDerivation() {
if (c.getEnd() - c.getStart() < minLength)
return null;
if (maxLength != NO_MAXIMUM && c.getEnd() - c.getStart() > maxLength)
return null;
return new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation();
}
};
}
@Override
public List<Derivation> call(Example ex, Callable c) {
if (c.getEnd() - c.getStart() < minLength)
return Collections.emptyList();
return Collections.singletonList(
new Derivation.Builder()
.withCallable(c)
.withFormulaFrom(c.child(0))
.createDerivation());
}
}

65
src/edu/stanford/nlp/sempre/FloatingFeatureComputer.java Normal file
View File

@ -0,0 +1,65 @@
package edu.stanford.nlp.sempre;
import java.util.*;
import fig.basic.IntPair;
/**
* Extract features specific to the floating parser.
*
* @author ppasupat
*/
public class FloatingFeatureComputer implements FeatureComputer {
@Override
public void extractLocal(Example ex, Derivation deriv) {
extractFloatingRuleFeatures(ex, deriv);
extractFloatingSkipFeatures(ex, deriv);
}
// Conjunction of the rule and each lemma in the sentence
void extractFloatingRuleFeatures(Example ex, Derivation deriv) {
if (!FeatureExtractor.containsDomain("floatRule")) return;
for (String lemma : ex.getLemmaTokens())
deriv.addFeature("floatRule", "lemma=" + lemma + ",rule=" + deriv.rule.toString());
}
// Look for words with no anchored rule applied
void extractFloatingSkipFeatures(Example ex, Derivation deriv) {
if (!FeatureExtractor.containsDomain("floatSkip")) return;
if (!deriv.isRoot(ex.numTokens())) return;
// Get all anchored tokens
boolean[] anchored = new boolean[ex.numTokens()];
List<Derivation> stack = new ArrayList<>();
stack.add(deriv);
while (!stack.isEmpty()) {
Derivation currentDeriv = stack.remove(stack.size() - 1);
if (deriv.start != -1) {
for (int i = deriv.start; i < deriv.end; i++)
anchored[i] = true;
} else {
for (Derivation child : currentDeriv.children) {
stack.add(child);
}
}
}
// Fire features based on tokens that are (not) anchored
// See if named entities are skipped
for (IntPair pair : ex.languageInfo.getNamedEntitySpans()) {
for (int i = pair.first; i < pair.second; i++) {
if (!anchored[i]) {
String nerTag = ex.languageInfo.nerTags.get(i);
deriv.addFeature("floatSkip", "skipped-ner=" + nerTag);
break;
}
}
}
// See which POS tags are skipped
for (int i = 0; i < anchored.length; i++) {
if (!anchored[i]) {
deriv.addFeature("floatSkip", "skipped-pos=" + ex.posTag(i));
}
}
}
}

357
src/edu/stanford/nlp/sempre/FloatingParser.java Normal file
View File

@ -0,0 +1,357 @@
package edu.stanford.nlp.sempre;
import fig.basic.*;
import java.util.*;
import static fig.basic.LogInfo.logs;
/**
* A FloatingParser builds Derivations according to a Grammar without having to
* generate the input utterance. In contrast, a conventional chart parser (e.g.,
* BeamParser) constructs parses for each span of the utterance. This is very
* inefficient when we're performing a more extractive semantic parsing task
* where many of the words are unaccounted for.
*
* Assume the Grammar is binarized and only has rules of the following form:
* $Cat => token
* $Cat => $Cat
* $Cat => token token
* $Cat => token $Cat
* $Cat => $Cat token
* $Cat => $Cat $Cat
* Each rule is either anchored or floating or both.
*
* Chart cells are either:
* - anchored: (cat, start, end) [these are effectively at depth 0]
* - floating: (cat, depth) [depends on anchored cells as base cases]
*
* Rules:
* cat => cat1 cat2 [binary]
* cat => cat1 [unary]
* Combinations:
* (cat1, start, end) => (cat, start, end)
* (cat1, depth) => (cat, depth)
*
* (cat1, start, mid), (cat2, mid, end) => (cat, start, end)
* (cat1, start, end), (cat2, depth) => (cat, depth + 1)
* (cat1, depth), (cat2, start, end) => (cat, depth + 1)
* (cat1, depth1), (cat2, depth2) => (cat, max(depth1, depth2)+1)
*
* @author Percy Liang
*/
public class FloatingParser extends Parser {
public static class Options {
@Option public int maxDepth = 10;
@Option public boolean defaultIsFloating = true;
@Option (gloss = "Flag specifying whether anchored spans/tokens can only be used once in a derivation")
public boolean useAnchorsOnce = false;
}
public static Options opts = new Options();
public FloatingParser(Spec spec) { super(spec); }
public ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts) {
return new FloatingParserState(this, params, ex, computeExpectedCounts);
}
}
/**
* Stores FloatingParser information about parsing a particular example. The actual
* parsing code lives here.
*
* Currently, many of the fields in ParserState are not used (chart).
* Those should be refactored out.
*
* @author Percy Liang
*/
class FloatingParserState extends ParserState {
// cell => list of derivations, formula set
// Examples of state:
// (category, depth)
// (category, depth, set of tokens)
private final Map<Object, List<Derivation>> chart = new HashMap<>();
public FloatingParserState(FloatingParser parser, Params params, Example ex, boolean computeExpectedCounts) {
super(parser, params, ex, computeExpectedCounts);
}
// Construct state names.
private Object floatingCell(String cat, int depth) {
return cat + ":" + depth;
}
private Object anchoredCell(String cat, int start, int end) {
return cat + "[" + start + "," + end + "]";
}
private Object cell(String cat, int start, int end, int depth) {
return (start != -1) ? anchoredCell(cat, start, end) : floatingCell(cat, depth);
}
private void addToChart(Object cell, Derivation deriv) {
if (Parser.opts.verbose >= 3)
LogInfo.logs("addToChart %s: %s", cell, deriv);
if (!deriv.isFeaturizedAndScored()) // A derivation could be belong in multiple cells.
featurizeAndScoreDerivation(deriv);
MapUtils.addToList(chart, cell, deriv);
}
private void applyRule(Rule rule, int start, int end, int depth, Derivation child1, Derivation child2, String canonicalUtterance) {
if (Parser.opts.verbose >= 5) logs("applyRule %s [%s:%s] depth=%s, %s %s", rule, start, end, depth, child1, child2);
List<Derivation> children;
if (child1 == null) // 0-ary
children = Collections.emptyList();
else if (child2 == null) // 1-ary
children = Collections.singletonList(child1);
else {
children = ListUtils.newList(child1, child2);
// optionally: ensure that specific anchors are only used once per final derivation
if (FloatingParser.opts.useAnchorsOnce &&
FloatingRuleUtils.derivationAnchorsOverlap(child1, child2))
return;
}
DerivationStream results = rule.sem.call(ex,
new SemanticFn.CallInfo(rule.lhs, start, end, rule, children));
while (results.hasNext()) {
Derivation newDeriv = results.next();
newDeriv.canonicalUtterance = canonicalUtterance;
// Avoid repetitive floating cells
/*
if (depth > 0) {
if (MapUtils.getSet(setChart, rule.lhs).contains(newDeriv.formula)) {
continue;
}
if (rule.lhs == Rule.rootCat)
LogInfo.logs("adding %s to root for floating", newDeriv);
MapUtils.addToSet(setChart, rule.lhs, newDeriv.formula);
}
*/
addToChart(cell(rule.lhs, start, end, depth), newDeriv);
if (depth == -1) // In addition, anchored cells become floating at level 0
addToChart(floatingCell(rule.lhs, 0), newDeriv);
}
}
private void applyAnchoredRule(Rule rule, int start, int end, Derivation child1, Derivation child2, String canonicalUtterance) {
applyRule(rule, start, end, -1, child1, child2, canonicalUtterance);
}
private void applyFloatingRule(Rule rule, int depth, Derivation child1, Derivation child2, String canonicalUtterance) {
applyRule(rule, -1, -1, depth, child1, child2, canonicalUtterance);
}
private List<Derivation> getDerivations(Object cell) {
List<Derivation> derivations = chart.get(cell);
// logs("getDerivations %s => %s", cell, derivations);
if (derivations == null) return Derivation.emptyList;
return derivations;
}
// Build derivations over span |start|, |end|.
private void buildAnchored(int start, int end) {
// Apply unary tokens on spans (rule $A (a))
for (Rule rule : parser.grammar.rules) {
if (!rule.isAnchored()) continue;
if (rule.rhs.size() != 1 || rule.isCatUnary()) continue;
boolean match = (end - start == 1) && ex.token(start).equals(rule.rhs.get(0));
if (match)
applyAnchoredRule(rule, start, end, null, null, rule.rhs.get(0));
}
// Apply binaries on spans (rule $A ($B $C)), ...
for (int mid = start + 1; mid < end; mid++) {
for (Rule rule : parser.grammar.rules) {
if (!rule.isAnchored()) continue;
if (rule.rhs.size() != 2) continue;
String rhs1 = rule.rhs.get(0);
String rhs2 = rule.rhs.get(1);
boolean match1 = (mid - start == 1) && ex.token(start).equals(rhs1);
boolean match2 = (end - mid == 1) && ex.token(mid).equals(rhs2);
if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) { // token $Cat
if (match1) {
List<Derivation> derivations = getDerivations(anchoredCell(rhs2, mid, end));
for (Derivation deriv : derivations)
applyAnchoredRule(rule, start, end, deriv, null, rhs1 + " " + deriv.canonicalUtterance);
}
} else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // $Cat token
if (match2) {
List<Derivation> derivations = getDerivations(anchoredCell(rhs1, start, mid));
for (Derivation deriv : derivations)
applyAnchoredRule(rule, start, end, deriv, null, deriv.canonicalUtterance + " " + rhs2);
}
} else if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // token token
if (match1 && match2)
applyAnchoredRule(rule, start, end, null, null, rhs1 + " " + rhs2);
} else { // $Cat $Cat
List<Derivation> derivations1 = getDerivations(anchoredCell(rhs1, start, mid));
List<Derivation> derivations2 = getDerivations(anchoredCell(rhs2, mid, end));
for (Derivation deriv1 : derivations1)
for (Derivation deriv2 : derivations2)
applyAnchoredRule(rule, start, end, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
}
}
}
// Apply unary categories on spans (rule $A ($B))
// Important: do this in topologically sorted order and after all the binaries are done.
for (Rule rule : parser.catUnaryRules) {
if (!rule.isAnchored()) continue;
List<Derivation> derivations = getDerivations(anchoredCell(rule.rhs.get(0), start, end));
for (Derivation deriv : derivations) {
applyAnchoredRule(rule, start, end, deriv, null, deriv.canonicalUtterance);
}
}
}
// Build floating derivations of exactly depth |depth|.
private void buildFloating(int depth) {
// Apply unary tokens on spans (rule $A (a))
if (depth == 1) {
for (Rule rule : parser.grammar.rules) {
if (!rule.isFloating()) continue;
if (rule.rhs.size() != 1 || rule.isCatUnary()) continue;
applyFloatingRule(rule, depth, null, null, rule.rhs.get(0));
}
}
// Apply binaries on spans (rule $A ($B $C)), ...
for (Rule rule : parser.grammar.rules) {
if (!rule.isFloating()) continue;
if (rule.rhs.size() != 2) continue;
String rhs1 = rule.rhs.get(0);
String rhs2 = rule.rhs.get(1);
if (!Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // token token
if (depth == 1)
applyFloatingRule(rule, depth, null, null, rhs1 + " " + rhs2);
} else if (!Rule.isCat(rhs1) && Rule.isCat(rhs2)) { // token $Cat
List<Derivation> derivations = getDerivations(floatingCell(rhs2, depth - 1));
for (Derivation deriv : derivations)
applyFloatingRule(rule, depth, deriv, null, rhs1 + " " + deriv.canonicalUtterance);
} else if (Rule.isCat(rhs1) && !Rule.isCat(rhs2)) { // $Cat token
List<Derivation> derivations = getDerivations(floatingCell(rhs1, depth - 1));
for (Derivation deriv : derivations)
applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance + " " + rhs2);
} else { // $Cat $Cat
for (int subDepth = 0; subDepth < depth; subDepth++) { // depth-1 <=depth-1
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, depth - 1));
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, subDepth));
for (Derivation deriv1 : derivations1)
for (Derivation deriv2 : derivations2)
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
}
for (int subDepth = 0; subDepth < depth - 1; subDepth++) { // <depth-1 depth-1
List<Derivation> derivations1 = getDerivations(floatingCell(rhs1, subDepth));
List<Derivation> derivations2 = getDerivations(floatingCell(rhs2, depth - 1));
for (Derivation deriv1 : derivations1)
for (Derivation deriv2 : derivations2)
applyFloatingRule(rule, depth, deriv1, deriv2, deriv1.canonicalUtterance + " " + deriv2.canonicalUtterance);
}
}
}
// Apply unary categories on spans (rule $A ($B))
// Important: do this in topologically sorted order and after all the binaries are done.
for (Rule rule : parser.catUnaryRules) {
if (!rule.isFloating()) continue;
List<Derivation> derivations = getDerivations(floatingCell(rule.rhs.get(0), depth - 1));
for (Derivation deriv : derivations)
applyFloatingRule(rule, depth, deriv, null, deriv.canonicalUtterance);
}
}
void addToDerivations(Object cell, List<Derivation> derivations) {
List<Derivation> myDerivations = chart.get(cell);
if (myDerivations != null)
derivations.addAll(myDerivations);
}
@Override public void infer() {
LogInfo.begin_track("FloatingParser.infer()");
// Base case ($TOKEN, $PHRASE)
for (Derivation deriv : gatherTokenAndPhraseDerivations()) {
addToChart(anchoredCell(deriv.cat, deriv.start, deriv.end), deriv);
addToChart(floatingCell(deriv.cat, 0), deriv);
}
Set<String> categories = new HashSet<>();
for (Rule rule : parser.grammar.rules)
categories.add(rule.lhs);
// Build up anchored derivations (like the BeamParser)
int numTokens = ex.numTokens();
for (int len = 1; len <= numTokens; len++) {
for (int i = 0; i + len <= numTokens; i++) {
buildAnchored(i, i + len);
for (String cat : categories) {
String cell = anchoredCell(cat, i, i + len).toString();
/*if (cat.equals("$Word") || cat.equals("$Fragment")) {
LogInfo.logs("Looking at cell: %s", cell);
if (chart.get(cell) == null) LogInfo.logs("???");
else {
for (Derivation deriv : chart.get(cell))
LogInfo.logs("Found cell of interest: %s", deriv.toString());
}
}*/
pruneCell(cell, chart.get(cell));
}
}
}
// Build up floating derivations
for (int depth = 1; depth <= FloatingParser.opts.maxDepth; depth++) {
buildFloating(depth);
for (String cat : categories) {
String cell = floatingCell(cat, depth).toString();
/*if (cat.equals("$Word") || cat.equals("$Fragment")) {
LogInfo.logs("Looking at floating cell: %s", cell);
if (chart.get(cell) == null) LogInfo.logs("???");
else {
for (Derivation deriv : chart.get(cell))
LogInfo.logs("Found cell of interest: %s", deriv.toString());
}
}*/
pruneCell(cell, chart.get(cell));
}
}
// Collect final predicted derivations
addToDerivations(anchoredCell(Rule.rootCat, 0, numTokens), predDerivations);
for (int depth = 1; depth <= FloatingParser.opts.maxDepth; depth++)
addToDerivations(floatingCell(Rule.rootCat, depth), predDerivations);
// Compute gradient with respect to the predicted derivations
ensureExecuted();
if (computeExpectedCounts) {
expectedCounts = new HashMap<>();
ParserState.computeExpectedCounts(predDerivations, expectedCounts);
}
// Example summary
if (Parser.opts.verbose >= 1) {
LogInfo.begin_track("Summary of Example %s", ex.getUtterance());
for (Derivation deriv : predDerivations)
LogInfo.logs("Generated: canonicalUtterance=%s, value=%s", deriv.canonicalUtterance, deriv.value);
LogInfo.end_track();
}
LogInfo.end_track();
}
private void visualizeAnchoredChart(Set<String> categories) {
for (String cat : categories) {
for (int len = 1; len <= numTokens; ++len) {
for (int i = 0; i + len <= numTokens; ++i) {
List<Derivation> derivations = getDerivations(anchoredCell(cat, i, i + len));
for (Derivation deriv : derivations) {
LogInfo.logs("ParserState.visualize: %s(%s:%s): %s", cat, i, i + len, deriv);
}
}
}
}
}
}

44
src/edu/stanford/nlp/sempre/FloatingRuleUtils.java Normal file
View File

@ -0,0 +1,44 @@
package edu.stanford.nlp.sempre;
import java.util.*;
/**
* Utilities for floating rules.
*/
public final class FloatingRuleUtils {
private FloatingRuleUtils() { } // Should not be called.
/**
* Get the anchored sub-derivations (sub-trees) of a derivation.
* I.e., gets all sub-derivations that are associated with a span of the utterance.
*/
public static List<Derivation> getDerivationAnchors(Derivation deriv) {
List<Derivation> anchors = new ArrayList<>();
if (deriv.rule.isAnchored()) {
// if the sub-derivation is anchored to a span just add it
anchors.add(deriv);
} else if (!(deriv.children == null || deriv.children.size() == 0)) {
// if the derivation is not anchored but has children, recurse into children
for (Derivation child : deriv.children)
anchors.addAll(getDerivationAnchors(child));
}
return anchors;
}
/**
* Helper function to ensure that anchored spans are only used once in a final derivation.
* for example if A spans (or has a child that spans) [0, 3] and B spans (or has a child
* that spans) [2, 4] then we have an overlap.
*/
public static boolean derivationAnchorsOverlap(Derivation a, Derivation b) {
List<Derivation> aRoots = getDerivationAnchors(a);
List<Derivation> bRoots = getDerivationAnchors(b);
for (Derivation aRoot : aRoots) {
for (Derivation bRoot : bRoots) {
if (aRoot.start < bRoot.end && bRoot.start < aRoot.end)
return true;
}
}
return false;
}
}

18
src/edu/stanford/nlp/sempre/Formula.java
View File

@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonValue;
import com.google.common.base.Function;
import fig.basic.LispTree;
import java.util.List;
/**
* A Formula is a logical form, which is the result of semantic parsing. Current
* implementation is lambda calculus with primitives like description logic and
@ -16,8 +18,8 @@ import fig.basic.LispTree;
* @author Percy Liang
*/
public abstract class Formula {
//cache the hashcode
private int hashCode=-1;
// cache the hashcode
private int hashCode = -1;
// Serialize as LispTree.
public abstract LispTree toLispTree();
@ -25,6 +27,10 @@ public abstract class Formula {
// Apply to formulas. If |func| returns null, then recurse on children.
public abstract Formula map(Function<Formula, Formula> func);
// Recursively perform some operation on each formula.
// Apply to formulas. If |func| returns an empty set or |alwaysRecurse|, then recurse on children.
public abstract List<Formula> mapToList(Function<Formula, List<Formula>> func, boolean alwaysRecurse);
@JsonValue
public String toString() { return toLispTree().toString(); }
@ -33,14 +39,14 @@ public abstract class Formula {
return Formulas.fromLispTree(LispTree.proto.parseFromString(str));
}
@Override abstract public boolean equals(Object o);
@Override public abstract boolean equals(Object o);
@Override public int hashCode() {
if(hashCode==-1)
if (hashCode == -1)
hashCode = computeHashCode();
return hashCode;
}
abstract public int computeHashCode();
public abstract int computeHashCode();
public static Formula nullFormula = new PrimitiveFormula() {
public LispTree toLispTree() { return LispTree.proto.newLeaf("null"); }

Some files were not shown because too many files have changed in this diff Show More