diff --git a/.gitignore b/.gitignore index aad22f6..fc18946 100644 --- a/.gitignore +++ b/.gitignore @@ -4,19 +4,12 @@ sfig refdb virtuoso-opensource +classes libsempre sempre.jar module-classes.txt -pystig -trans - -classes -libsempre - state -lex_state -trans_state out test-output diff --git a/freebase/data/debug-test.grammar b/freebase/data/debug-test.grammar deleted file mode 100644 index e47aa27..0000000 --- a/freebase/data/debug-test.grammar +++ /dev/null @@ -1,30 +0,0 @@ -# For debugging - -#Entities -(rule $Entity (barack obama) (ConstantFn fb:en.barack_obama fb:people.person)) -(rule $Entity (michelle obama) (ConstantFn fb:en.michelle_obama fb:people.person)) -(rule $Entity (tom cruise) (ConstantFn fb:en.tom_cruise fb:people.person)) -(rule $Entity (michael jordan) (ConstantFn fb:en.michael_jordan fb:people.person)) -(rule $Entity (michael jackson) (ConstantFn fb:en.michael_jackson fb:people.person)) -(rule $Entity (thomas edison) (ConstantFn fb:en.thomas_edison fb:people.person)) -(rule $Entity (albert einstein) (ConstantFn fb:en.albert_einstein fb:people.person)) -(rule $Entity (edgar allan poe) (ConstantFn fb:en.edgar_allan_poe fb:people.person)) -(rule $Entity (lionel messi) (ConstantFn fb:en.lionel_messi fb:people.person)) -(rule $Entity (alan turing) (ConstantFn fb:en.alan_turing fb:people.person)) -(rule $Entity (abraham lincoln) (ConstantFn fb:en.abraham_lincoln fb:people.person)) -(rule $Entity (steve jobs) (ConstantFn fb:en.steve_jobs fb:people.person)) - -#Binaries -(rule $Binary (born) (ConstantFn fb:people.person.place_of_birth (-> fb:location.location fb:people.person))) -(rule $Binary (profession) (ConstantFn fb:people.person.profession (-> fb:people.profession fb:people.person))) -(rule $Binary (lived) (ConstantFn (lambda x (fb:people.person.places_lived (fb:people.place_lived.location (var x)))) (-> fb:location.location fb:people.person))) -(rule $Binary (studied) (ConstantFn (lambda x (fb:people.person.education (fb:education.education.institution (var x)))) (-> fb:education.educational_institution fb:people.person))) -(rule $Binary (worked) (ConstantFn (lambda x (fb:people.person.employment_history (fb:business.employment_tenure.company (var x)))) (-> fb:business_employer fb:people.person))) -(rule $Binary (nationality) (ConstantFn fb:people.person.nationality (-> fb:location.country fb:people.person))) -(rule $Binary (language) (ConstantFn fb:people.person.languages (-> fb:language.human_language fb:people.person))) -(rule $Binary (birthplace) (ConstantFn fb:people.person.place_of_birth (-> fb:location.location fb:people.person))) -(rule $Binary (parents) (ConstantFn fb:people.person.parents (-> fb:people.person fb:people.person))) -(rule $Binary (children) (ConstantFn fb:people.person.children (-> fb:people.person fb:people.person))) - -#Combine -(rule $ROOT ($Entity $Binary) (JoinFn unary,binary unaryCanBeArg0)) diff --git a/freebase/data/formal.grammar b/freebase/data/formal.grammar deleted file mode 100644 index e8baad8..0000000 --- a/freebase/data/formal.grammar +++ /dev/null @@ -1,52 +0,0 @@ -# Grammar that combines formal language (for composition) with natural language -# (for lexical triggering). Primarily used to debug and to build queries -# semi-automatically. -# -# Examples: -# [[b birthplace] [e obama]] -# [[e obama] [b born]] -# [[[e spanish] [b speak]] [u country]] -# [number of [[u lake] [[b in] [e united states]]]] -# [br [e tom cruise] [u movies]] -# [most [b height] [u president]] -# [[u president] [[b height] [at least 1.9]]] - -### Primitives - -(rule $Entity ([ e $PHRASE ]) (LexiconFn entity)) -(rule $Unary ([ u $LEMMA_PHRASE ]) (LexiconFn unary)) -(rule $Binary ([ b $LEMMA_PHRASE ]) (LexiconFn binary)) - -(rule $Binary (more) (ConstantFn > (union (-> fb:type.int fb:type.int) (-> fb:type.float fb:type.float)))) -(rule $Binary (less) (ConstantFn < (union (-> fb:type.int fb:type.int) (-> fb:type.float fb:type.float)))) -(rule $Binary (at least) (ConstantFn >= (union (-> fb:type.int fb:type.int) (-> fb:type.float fb:type.float)))) -(rule $Binary (at most) (ConstantFn <= (union (-> fb:type.int fb:type.int) (-> fb:type.float fb:type.float)))) - -(rule $Set ($PHRASE) (NumberFn)) -(rule $Set ($Entity) (IdentityFn)) -(rule $Set ($Unary) (IdentityFn)) - -### Operators - -# Aggregation -(rule $Operator (count) (ConstantFn (lambda x (count (var x))))) - -# Superlative -(rule $ArgmaxStr (most) (ConstantFn null null)) -(rule $Operator2 ($ArgmaxStr) (ConstantFn (lambda degree (lambda head (argmax 1 1 (var head) (var degree)))) - (-> (-> fb:common.topic fb:type.number) fb:common.topic fb:common.topic))) -(rule $Operator ($Operator2 $Binary) (JoinFn binary,unary unaryCanBeArg1)) - -(rule $Set ([ $Operator $Set ]) (JoinFn binary,unary unaryCanBeArg1)) - -### Composition - -(rule $Set ([ $Binary $Set ]) (JoinFn binary,unary unaryCanBeArg0 unaryCanBeArg1)) -(rule $Set ([ $Set $Binary ]) (JoinFn unary,binary unaryCanBeArg0 unaryCanBeArg1)) -(rule $Set ([ $Set $Set ]) (MergeFn and)) -(rule $Set ([ br $Set $Set ]) (BridgeFn unary headFirst)) -(rule $Set ([ br $Set $Set ]) (BridgeFn unary headLast)) - -(rule $ROOT ($Set) (IdentityFn)) -(rule $ROOT ($Binary) (IdentityFn)) -(rule $ROOT ($Operator) (IdentityFn)) diff --git a/freebase/data/freebase-demo.grammar b/freebase/data/freebase-demo.grammar deleted file mode 100644 index e0fd978..0000000 --- a/freebase/data/freebase-demo.grammar +++ /dev/null @@ -1,159 +0,0 @@ -# Simple rule-based system for Freebase QA. -# Still needs some parameter tuning due to ambiguity. - -(def @type fb:type.object.type) -(def @male (fb:people.person.gender fb:en.male)) -(def @female (fb:people.person.gender fb:en.female)) -(def @containedby (lambda x (or (fb:location.location.containedby (var x)) (fb:location.location.partially_containedby (var x))))) - -(when (not strict) - (rule $NamedEntity ($PHRASE) (FilterPosTagFn span NN NNS NNP NNPS)) -) -(rule $NamedEntity ($PHRASE) (FilterNerSpanFn PERSON ORGANIZATION LOCATION MISC)) -(rule $Entity ($NamedEntity) (LexiconFn entity fbsearch)) -(rule $Set ($Entity) (IdentityFn)) - -# Easy properties whose directionality can be inferred -(for @x (language languages speak spoken) (rule $Property (@x) (ConstantFn fb:location.country.official_language))) -(for @x (book books write wrote written) (rule $Property (@x) (ConstantFn fb:book.author.works_written))) - -(for @x (parents) (rule $Property (@x) (ConstantFn fb:people.person.parents))) -(for @x (mother mom) (rule $Property (@x) (ConstantFn (lambda x (fb:people.person.parents (and (var x) @female)))))) -(for @x (father dad) (rule $Property (@x) (ConstantFn (lambda x (fb:people.person.parents (and (var x) @male)))))) -(for @x (sibling siblings) (rule $Property (@x) (ConstantFn (lambda x (and (!= (var x)) (fb:people.person.sibling_s (fb:people.sibling_relationship.sibling (var x)))))))) -(for @x (sister sisters) (rule $Property (@x) (ConstantFn (lambda x (and (!= (var x)) (fb:people.person.sibling_s (fb:people.sibling_relationship.sibling (and (var x) @female)))))))) -(for @x (brother brothers) (rule $Property (@x) (ConstantFn (lambda x (and (!= (var x)) (fb:people.person.sibling_s (fb:people.sibling_relationship.sibling (and (var x) @male)))))))) -(for @x (child children kid kids) (rule $Property (@x) (ConstantFn fb:people.person.children))) -(for @x (son sons) (rule $Property (@x) (ConstantFn (lambda x (fb:people.person.children (and (var x) @male)))))) -(for @x (daughter daughters) (rule $Property (@x) (ConstantFn (lambda x (fb:people.person.children (and (var x) @female)))))) -(for @x (spouse marry married marrying wife husband) (rule $Property (@x) (ConstantFn (lambda x (and (!= (var x)) (fb:people.person.spouse_s (fb:people.marriage.spouse (var x)))))))) - -(for @x (tall height) (rule $Property (@x) (ConstantFn fb:people.person.height_meters))) -(for @x (tall height elevation) (rule $Property (@x) (ConstantFn fb:geography.mountain.elevation))) -(for @x (long length) (rule $Property (@x) (ConstantFn fb:geography.river.length))) -(for @x (deep depth) (rule $Property (@x) (ConstantFn fb:geography.body_of_water.depth))) -(for @x ((live) (lived)) (rule $Property @x (ConstantFn (lambda x (fb:people.person.places_lived (fb:people.place_lived.location (var x))))))) -(for @x ((founded) (founder) (founders)) (rule $Property @x (ConstantFn fb:organization.organization.founders))) -(for @x ((president)) (rule $Property @x (ConstantFn (lambda x (fb:government.governmental_jurisdiction.governing_officials (and (fb:government.government_position_held.basic_title fb:en.president) (fb:government.government_position_held.office_holder (var x)))))))) -(for @x ((school) (schools) (education) (study)) (rule $Property @x (ConstantFn (lambda x (fb:people.person.education (fb:education.education.institution (var x))))))) -(for @x ((college) (colleges) (university) (universities)) (rule $Property @x (ConstantFn (lambda x (fb:people.person.education (fb:education.education.institution (and (var x) (@type fb:education.university)))))))) -(for @x ((win) (won) (award) (awards)) (rule $Property @x (ConstantFn (lambda x (fb:award.award_winner.awards_won (fb:award.award_honor.award (var x))))))) -(for @x ((win) (won)) (rule $Property @x (ConstantFn fb:sports.sports_team.championships))) -(for @x ((border) (borders) (bordering)) (rule $Property @x (ConstantFn (lambda x (fb:location.location.adjoin_s (fb:location.adjoining_relationship.adjoins (var x))))))) -(for @x ((style) (genre)) (rule $Property @x (ConstantFn fb:music.artist.genre))) -(for @x ((style) (genre)) (rule $Property @x (ConstantFn fb:visual_art.visual_artist.associated_periods_or_movements))) - -(for @x ((government) (political system)) (rule $Property @x (ConstantFn fb:location.country.form_of_government))) -(for @x ((attractions) (to do) (to see)) (rule $Property @x (ConstantFn fb:travel.travel_destination.tourist_attractions))) - -(for @x ((river) (rivers)) (rule $Unary @x (ConstantFn (@type fb:geography.river)))) -(for @x ((lake) (lakes)) (rule $Unary @x (ConstantFn (@type fb:geography.lake)))) -(for @x ((mountain) (mountains)) (rule $Unary @x (ConstantFn (@type fb:geography.mountain)))) -(for @x ((city) (cities)) (rule $Unary @x (ConstantFn (@type fb:location.citytown)))) -(for @x ((state) (states)) (rule $Unary @x (ConstantFn (@type fb:location.us_state)))) -(rule $Property ($Unary) (lambda u (lambda x (fb:location.location.contains (and (var x) (var u)))))) - -(for @x (continent) (rule $Property (@x) (ConstantFn (lambda x (@containedby (and (var x) (@type fb:location.continent))))))) -(for @x (county) (rule $Property (@x) (ConstantFn (lambda x (@containedby (and (var x) (@type fb:location.us_county))))))) -(for @x (country) (rule $Property (@x) (ConstantFn (lambda x (@containedby (and (var x) (@type fb:location.country))))))) -(for @x (located) (rule $Property (@x) (ConstantFn @containedby))) -(for @x (capital) (rule $Property (@x) (ConstantFn fb:location.country.capital))) -(for @x (capital) (rule $Property (@x) (ConstantFn fb:location.us_state.capital))) - -(for @x ((currency) (money)) (rule $Property @x (ConstantFn fb:location.country.currency_used))) -(for @x ((timezone) (timezones) (time zone) (time zones)) (rule $Property @x (ConstantFn fb:location.location.time_zones))) -(for @x ((zipcode) (zipcodes) (zip code) (zip codes)) (rule $Property @x (ConstantFn fb:location.citytown.postal_codes))) -(for @x ((die of) (die from) (died of) (cause of death) (died from)) (rule $Property @x (ConstantFn fb:people.deceased_person.cause_of_death))) -(for @x ((star in) (starred in) (movie) (movies) (film) (films)) (rule $Property @x (ConstantFn (lambda x (fb:film.actor.film (fb:film.performance.film (var x))))))) -(for @x ((play for) (team) (teams)) (rule $Property @x (ConstantFn (lambda x (fb:sports.pro_athlete.teams (fb:sports.sports_team_roster.team (var x))))))) -(for @x ((invention) (inventions) (invent) (invented) (inventor)) (rule $Property @x (ConstantFn fb:law.invention.inventor))) -(for @x ((religion) (beliefs) (believe in)) (rule $Property @x (ConstantFn fb:people.person.religion))) -(for @x ((instrument) (play)) (rule $Property @x (ConstantFn fb:music.group_member.instruments_played))) -(for @x ((party)) (rule $Property @x (ConstantFn (lambda x (fb:government.politician.party (fb:government.political_party_tenure.party (var x))))))) -(for @x ((drafted) (start) (started)) (rule $Property @x (ConstantFn fb:sports.pro_athlete.career_start))) -(for @x ((headquarters)) (rule $Property @x (ConstantFn (lambda x (fb:organization.organization.headquarters (fb:location.mailing_address.country (var x))))))) -(for @x ((tv)) (rule $Property @x (ConstantFn (lambda x (fb:tv.tv_actor.starring_roles (fb:tv.regular_tv_appearance.series (var x))))))) - -(rule $Query (($PHRASE optional) $Property ($PHRASE optional) $Set ($PHRASE optional)) (JoinFn unaryCanBeArg0 unaryCanBeArg1 binary,unary)) -(rule $Query (($PHRASE optional) $Set ($PHRASE optional) $Property ($PHRASE optional)) (JoinFn unaryCanBeArg0 unaryCanBeArg1 unary,binary)) - -# Typed by where and when -(for @x ((founded)) (rule $WhenProperty @x (ConstantFn fb:organization.organization.date_founded))) -(for @x ((founded)) (rule $WhereProperty @x (ConstantFn fb:organization.organization.place_founded))) -(for @x ((born)) (rule $WhenProperty @x (ConstantFn fb:people.person.date_of_birth))) -(for @x ((born) (grow up) (from)) (rule $WhereProperty @x (ConstantFn fb:people.person.place_of_birth))) -(for @x ((die) (died)) (rule $WhenProperty @x (ConstantFn fb:people.deceased_person.date_of_death))) -(for @x ((die) (died)) (rule $WhereProperty @x (ConstantFn fb:people.deceased_person.place_of_death))) -(for @x ((play)) (rule $WhereProperty @x (ConstantFn fb:sports.sports_team.arena_stadium))) - -(for @x ((when) (what year) (what date)) (rule $When @x (ConstantFn null))) -(for @x ((where) (what city) (what country)) (rule $Where @x (ConstantFn null))) -(rule $WhenSet ($When ($PHRASE optional) $Set) (SelectFn 1)) -(rule $WhereSet ($Where ($PHRASE optional) $Set) (SelectFn 1)) -(when (not strict) - (rule $ROOT ($WhenSet ($PHRASE optional) $WhenProperty ($PHRASE optional)) (JoinFn unaryCanBeArg0 unary,binary)) - (rule $ROOT ($WhereSet ($PHRASE optional) $WhereProperty ($PHRASE optional)) (JoinFn unaryCanBeArg0 unary,binary)) -) - -# fb:time.event.start_date - -# Properties that can only go at the beginning or end -(for @x ((who was)) (rule $BeginProperty @x (ConstantFn fb:people.person.profession))) -(for @x ((occupation) (known for) (famous for)) (rule $Property @x (ConstantFn fb:people.person.profession))) -(for @x ((known for) (famous for)) (rule $Property @x (ConstantFn fb:base.argumentmaps.innovator.original_ideas))) -(for @x ((do) (do for a living) (doing now)) (rule $EndProperty @x (ConstantFn fb:people.person.profession))) - -(when (not strict) - (rule $ROOT (($PHRASE optional) $Set $EndProperty (? optional)) (JoinFn unaryCanBeArg0 unary,binary)) - (rule $ROOT ($BeginProperty $Set (? optional)) (JoinFn unaryCanBeArg0 binary,unary)) -) - -# Verbs (directionality is important) -(for @x ((influence) (influenced) (inspire) (inspired)) (rule $Verb @x (ConstantFn fb:influence.influence_node.influenced))) - -# Handle verbs -(rule $Query (($PHRASE optional) $Verb $Set ($PHRASE optional)) (JoinFn unaryCanBeArg1 binary,unary)) # who influenced van Gogh? -(rule $Query (($PHRASE optional) $Verb by $Set ($PHRASE optional)) (JoinFn unaryCanBeArg0 binary,unary)) # who was influenced by van Gogh? -(rule $Query (($PHRASE optional) $Set $Verb (? optional)) (JoinFn unaryCanBeArg0 unary,binary)) # who did van Gogh influence? -(rule $Query (($PHRASE optional) $Set $Verb by ($PHRASE optional)) (JoinFn unaryCanBeArg1 unary,binary)) # who was van Gogh influenced by? - -#(rule $ROOT ($Query) (IdentityFn)) - -# Counting -(when (not strict) - (rule $ROOT (how many $Query) (lambda x (count (var x)))) -) - -# Superlatives -(rule $Degree (biggest) (ConstantFn fb:location.location.area)) -(rule $Degree (largest) (ConstantFn fb:location.location.area)) -(rule $Property (population) (ConstantFn (lambda x (!fb:measurement_unit.dated_integer.number (argmax 1 1 (!fb:location.statistical_region.population (var x)) fb:measurement_unit.dated_integer.year))))) -(rule $Degree (largest) (ConstantFn (lambda x (fb:location.statistical_region.population (fb:measurement_unit.dated_integer.number (var x)))))) -(rule $Degree (tallest) (ConstantFn fb:people.person.height_meters)) -(rule $Degree (tallest) (ConstantFn fb:geography.mountain.elevation)) -(rule $Degree (longest) (ConstantFn fb:geography.river.length)) -(rule $Degree (deepest) (ConstantFn fb:geography.body_of_water.depth)) -(rule $Superlative ($Degree) (lambda d (lambda x (argmax 1 1 (var x) (var d))))) -(rule $Ordinal ($PHRASE) (NumberFn ORDINAL)) # second -(rule $Number ($PHRASE) (NumberFn NUMBER)) # two -(rule $Superlative ($Ordinal $Degree) (lambda n (lambda d (lambda x (argmax (var n) 1 (var x) (var d)))))) # second largest -(rule $Superlative ($Number $Degree) (lambda n (lambda d (lambda x (argmax 1 (var n) (var x) (var d)))))) # two largest - -# (execute (argmax 1 2 (and (and (@type fb:location.citytown) (@containedby fb:en.california)) (not (!fb:geography.river.cities (@type fb:geography.river)))) fb:location.location.area)) -(rule $Set ($Unary) (IdentityFn)) # river -(rule $Restrict (in ($Optional optional) $Set) (lambda x (@containedby (var x)))) # in California -(rule $Restrict (near ($Optional optional) $Set) (lambda x (!fb:geography.river.cities (var x)))) # near a river -(rule $Restrict (not $Restrict) (lambda x (not (var x)))) # not in California -(rule $Set ($Set ($Optional optional) $Restrict) (lambda x (lambda y (and (var x) (var y))))) - -(for @x (what is are was were a an the that) (rule $OptionalWord (@x) (ConstantFn null))) -(rule $Optional ($OptionalWord) (IdentityFn)) -(rule $Optional ($OptionalWord $Optional) (IdentityFn)) - -(rule $ROOT ($Set) (IdentityFn)) -(rule $ROOT (($Optional optional) $Superlative $Set (? optional)) (JoinFn forward betaReduce)) - -(for @x ((first winner) (first $PHRASE win)) (rule $Property @x (ConstantFn (lambda x (!fb:award.award_honor.award_winner (argmin 1 1 (fb:award.award_honor.award (var x)) fb:award.award_honor.year)))))) - -# Debugging -(rule $ROOT (e $Entity) (IdentityFn)) diff --git a/freebase/data/high-precision.grammar b/freebase/data/high-precision.grammar deleted file mode 100644 index d1d4bdd..0000000 --- a/freebase/data/high-precision.grammar +++ /dev/null @@ -1,9 +0,0 @@ -(rule $Entity ($PHRASE) (LexiconFn entity fbsearch)) -(rule $ROOT ($Entity born) (lambda x (!fb:people.person.place_of_birth (var x)))) - -#(rule $Set ($Entity) (IdentityFn)) -#(rule $VP/NP (wrote) (ConstantFn fb:book.author.works_written)) -#(rule $S (where was $Set born) (lambda x (!fb:people.person.place_of_birth (var x)))) - -#(rule $ROOT ($S) (IdentityFn)) -#(rule $ROOT (e $Entity) (IdentityFn)) diff --git a/freebase/data/jayant-emnlp2012-test.examples b/freebase/data/jayant-emnlp2012-test.examples deleted file mode 100644 index bba8e79..0000000 --- a/freebase/data/jayant-emnlp2012-test.examples +++ /dev/null @@ -1,250 +0,0 @@ -(example - (utterance "capital of Uzbekistan") - (targetFormula (!fb:location.country.capital fb:en.uzbekistan_russia)) -) - -(example - (utterance "capital of Serbia") - (targetFormula (!fb:location.country.capital fb:en.serbia)) -) - -(example - (utterance "city of Arizona") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.arizona))) -) - -(example - (utterance "home to North Dakota State University") - (targetFormula (and (fb:type.object.type fb:location.citytown) (!fb:location.location.containedby fb:en.north_dakota_state_university))) -) - -(example - (utterance "country in Asia") - (targetFormula (and (fb:type.object.type fb:location.country) (fb:location.location.containedby fb:en.asia))) -) - -(example - (utterance "part of California") - (targetFormula (fb:location.location.containedby fb:en.california)) -) - -(example - (utterance "city in Indonesia") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.indonesia))) -) - -(example - (utterance "resident of Morehead , Kentucky") - (targetFormula (fb:people.person.places_lived (fb:people.place_lived.location fb:en.morehead_kentucky))) -) - -(example - (utterance "singer of Evanescence") - (targetFormula (and (fb:people.person.profession fb:en.singer) (fb:music.group_member.membership (fb:music.group_membership.group fb:en.evanescence)))) -) - -(example - (utterance "capital of the Republic of Croatia") - (targetFormula (!fb:location.country.capital fb:en.croatia)) -) - -(example - (utterance "town in Austria") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.austria))) -) - -(example - (utterance "city in North Rhine-Westphalia , Germany") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.north_rhine-westphalia))) -) - -(example - (utterance "part of Iraq") - (targetFormula (fb:location.location.containedby fb:en.iraq)) -) - -(example - (utterance "state in Mexico") - (targetFormula (and (fb:type.object.type fb:location.administrative_division) (fb:location.location.containedby fb:en.mexico))) -) - -(example - (utterance "composer from Havana , Cuba") - (targetFormula (and (fb:type.object.type fb:music.composer) (fb:people.person.place_of_birth fb:en.havana))) -) - -(example - (utterance "city in Lombardy , Italy") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.lombardy))) -) - -(example - (utterance "search engine") - (targetFormula (and (fb:type.object.type fb:business.business_operation) (fb:business.business_operation.industry fb:en.search_engine))) -) - -(example - (utterance "woman in Japan") - (targetFormula (and (fb:people.person.gender fb:en.female) (fb:people.person.places_lived (fb:people.place_lived.location fb:en.japan)))) -) - -(example - (utterance "capital of Indonesia") - (targetFormula (!fb:location.country.capital fb:en.indonesia)) -) - -(example - (utterance "novel by Joanne Harris") - (targetFormula (!fb:book.author.works_written fb:en.joanne_harris)) -) - -(example - (utterance "Spanish-speaking country in South America") - (targetFormula (and (fb:type.object.type fb:location.country) (and (fb:location.location.containedby fb:en.south_america) (fb:location.country.official_language fb:en.spanish)))) -) - -(example - (utterance "capital city of Romania") - (targetFormula (!fb:location.country.capital fb:en.romania)) -) - -(example - (utterance "home of Michigan Technological University") - (targetFormula (and (fb:type.object.type fb:location.citytown) (!fb:location.location.containedby fb:en.michigan_technological_university))) -) - -(example - (utterance "congressman from Oklahoma") - (targetFormula (and (fb:type.object.type fb:government.u_s_congressperson) (fb:government.politician.government_positions_held (fb:government.government_position_held.district_represented fb:en.oklahoma)))) -) - -(example - (utterance "capital of Jordan") - (targetFormula (!fb:location.country.capital fb:en.jordan)) -) - -(example - (utterance "capital of the Republic of Serbia") - (targetFormula (!fb:location.country.capital fb:en.serbia)) -) - -(example - (utterance "capital city of Iceland") - (targetFormula (!fb:location.country.capital fb:en.iceland)) -) - -(example - (utterance "port of Italy") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.italy))) -) - -(example - (utterance "capital of Romania") - (targetFormula (!fb:location.country.capital fb:en.romania)) -) - -(example - (utterance "native of Marrero , Louisiana") - (targetFormula (fb:people.person.place_of_birth fb:en.marrero)) -) - -(example - (utterance "city in Jordan") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.jordan))) -) - -(example - (utterance "capital of Belarus") - (targetFormula (!fb:location.country.capital fb:en.belarus)) -) - -(example - (utterance "native of Miami , Florida") - (targetFormula (fb:people.person.place_of_birth fb:en.miami)) -) - -(example - (utterance "landmark of London") - (targetFormula (and (fb:type.object.type fb:travel.tourist_attraction) (fb:location.location.containedby fb:en.london))) -) - -(example - (utterance "book by Barbara Matthiessen") - (targetFormula (!fb:book.author.works_written fb:en.barbara_matthiessen)) -) - -(example - (utterance "city of Australia") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.australia))) -) - -(example - (utterance "town in South Ayrshire , Scotland") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.south_ayrshire))) -) - -(example - (utterance "capital city of Turkey") - (targetFormula (!fb:location.country.capital fb:en.turkey)) -) - -(example - (utterance "singer from Tashkent , Uzbekistan") - (targetFormula (and (fb:people.person.profession fb:en.singer) (fb:people.person.place_of_birth fb:en.tashkent))) -) - -(example - (utterance "writer in Kawaguchi , Japan") - (targetFormula (and (fb:type.object.type fb:book.author) (fb:people.person.places_lived (fb:people.place_lived.location fb:en.kawaguchi)))) -) - -(example - (utterance "native of Tuscon , Arizona") - (targetFormula (fb:people.person.place_of_birth fb:en.tucson)) -) - -(example - (utterance "city in Scotland") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.scotland))) -) - -(example - (utterance "center of Kazakhstan") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.kazakhstan))) -) - -(example - (utterance "river in Japan") - (targetFormula (and (fb:type.object.type fb:geography.river) (fb:location.location.containedby fb:en.japan))) -) - -(example - (utterance "market in Australia") - (targetFormula (and (fb:type.object.type fb:travel.tourist_attraction) (fb:location.location.containedby fb:en.australia))) -) - -(example - (utterance "capital of Guyana") - (targetFormula (!fb:location.country.capital fb:en.guyana)) -) - -(example - (utterance "son of Baltimore") - (targetFormula (fb:people.person.place_of_birth fb:en.baltimore)) -) - -(example - (utterance "ex-guitarist for Paramore") - (targetFormula (and (fb:people.person.profession fb:en.guitarist) (fb:music.group_member.membership (fb:music.group_membership.group fb:en.paramore)))) -) - -(example - (utterance "town in Okinawa , Japan") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.okinawa_prefecture))) -) - -(example - (utterance "capital of Tanzania") - (targetFormula (!fb:location.country.capital fb:en.tanzania)) -) - diff --git a/freebase/data/jayant-emnlp2012-validation.examples b/freebase/data/jayant-emnlp2012-validation.examples deleted file mode 100644 index b5a3a5c..0000000 --- a/freebase/data/jayant-emnlp2012-validation.examples +++ /dev/null @@ -1,250 +0,0 @@ -(example - (utterance "district of Sri Lanka") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.sri_lanka))) -) - -(example - (utterance "Spanish-speaking country in North America") - (targetFormula (and (fb:type.object.type fb:location.country) (and (fb:location.location.containedby fb:en.north_america) (fb:location.country.official_language fb:en.spanish)))) -) - -(example - (utterance "native of Houston , Texas") - (targetFormula (fb:people.person.place_of_birth fb:en.houston)) -) - -(example - (utterance "town in Malaysia") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.malaysia))) -) - -(example - (utterance "author of Dogs Misbehaving") - (targetFormula (fb:book.author.works_written fb:en.dogs_misbehaving)) -) - -(example - (utterance "languages of China") - (targetFormula (!fb:location.country.languages_spoken fb:en.china)) -) - -(example - (utterance "currency in Finland") - (targetFormula (!fb:location.country.currency_used fb:en.finland)) -) - -(example - (utterance "currency in Denmark") - (targetFormula (!fb:location.country.currency_used fb:en.denmark)) -) - -(example - (utterance "son of Nate Saint") - (targetFormula (and (fb:people.person.gender fb:en.male) (!fb:people.person.children fb:en.nate_saint))) -) - -(example - (utterance "country in Europe") - (targetFormula (and (fb:type.object.type fb:location.country) (fb:location.location.containedby fb:en.europe))) -) - -(example - (utterance "square in Bologna , Italy") - (targetFormula (and (fb:type.object.type fb:travel.tourist_attraction) (fb:location.location.containedby fb:en.bologna))) -) - -(example - (utterance "home to Skybus Airlines") - (targetFormula (!fb:location.mailing_address.citytown (!fb:organization.organization.headquarters fb:en.skybus_airlines))) -) - -(example - (utterance "city in Scotland") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.scotland))) -) - -(example - (utterance "capital of Indiana") - (targetFormula (!fb:location.administrative_division_capital_relationship.capital (!fb:location.administrative_division.capital fb:en.indiana))) -) - -(example - (utterance "capital of Russia") - (targetFormula (!fb:location.country.capital fb:en.russia)) -) - -(example - (utterance "resident of Dimondale , Michigan") - (targetFormula (fb:people.person.places_lived (fb:people.place_lived.location fb:en.dimondale))) -) - -(example - (utterance "school in Searcy") - (targetFormula (and (fb:type.object.type fb:education.school) (fb:organization.organization.headquarters (fb:location.mailing_address.citytown fb:en.searcy)))) -) - -(example - (utterance "point in Finland") - (targetFormula (fb:location.location.containedby fb:en.finland)) -) - -(example - (utterance "city in Canada") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.canada))) -) - -(example - (utterance "home to Fairfield University") - (targetFormula (and (fb:type.object.type fb:location.citytown) (!fb:location.location.containedby fb:en.fairfield_university))) -) - -(example - (utterance "country in Africa") - (targetFormula (and (fb:type.object.type fb:location.country) (fb:location.location.containedby fb:en.africa))) -) - -(example - (utterance "city in South Dakota") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.south_dakota))) -) - -(example - (utterance "vocalist from London , England") - (targetFormula (and (fb:people.person.profession fb:en.singer) (fb:people.person.place_of_birth fb:en.london))) -) - -(example - (utterance "capital of Peru") - (targetFormula (!fb:location.country.capital fb:en.peru)) -) - -(example - (utterance "alumni of Regis University in Denver") - (targetFormula (fb:people.person.education (fb:education.education.institution fb:en.regis_university))) -) - -(example - (utterance "part of North Carolina") - (targetFormula (fb:location.location.containedby fb:en.north_carolina)) -) - -(example - (utterance "spot in Central America") - (targetFormula (fb:location.location.containedby fb:en.central_america)) -) - -(example - (utterance "city of Rembrandt") - (targetFormula (and (fb:type.object.type fb:location.citytown) (!fb:people.person.place_of_birth fb:en.rembrandt))) -) - -(example - (utterance "capital of Jamaica") - (targetFormula (!fb:location.country.capital fb:en.jamaica)) -) - -(example - (utterance "native of Buenos Aires") - (targetFormula (fb:people.person.place_of_birth fb:en.buenos_aires)) -) - -(example - (utterance "shrine in Texas") - (targetFormula (and (fb:type.object.type fb:religion.place_of_worship) (fb:location.location.containedby fb:en.texas))) -) - -(example - (utterance "native of Johnson City , Tennessee") - (targetFormula (fb:people.person.place_of_birth fb:en.johnson_city)) -) - -(example - (utterance "capital of Nigeria") - (targetFormula (!fb:location.country.capital fb:en.nigeria)) -) - -(example - (utterance "capital city of Venezuela") - (targetFormula (!fb:location.country.capital fb:en.venezuela)) -) - -(example - (utterance "capital of Denmark") - (targetFormula (!fb:location.country.capital fb:en.denmark)) -) - -(example - (utterance "currency in Tunisia") - (targetFormula (!fb:location.country.currency_used fb:en.tunisia)) -) - -(example - (utterance "son of Abraham and Hagar") - (targetFormula (and (fb:people.person.gender fb:en.male) (and (!fb:people.person.children fb:en.abraham) (!fb:people.person.children fb:en.hagar)))) -) - -(example - (utterance "capital of England") - (targetFormula (!fb:location.country.capital fb:en.england)) -) - -(example - (utterance "stadium in Fullerton , California") - (targetFormula (and (fb:type.object.type fb:sports.sports_facility) (fb:location.location.containedby fb:en.fullerton))) -) - -(example - (utterance "cities in Indonesia") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.indonesia))) -) - -(example - (utterance "town near Koblenz , Germany") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.koblenz))) -) - -(example - (utterance "home to company called Microsoft") - (targetFormula (!fb:location.mailing_address.citytown (!fb:organization.organization.headquarters fb:en.microsoft_corporation))) -) - -(example - (utterance "currency for Finland") - (targetFormula (!fb:location.country.currency_used fb:en.finland)) -) - -(example - (utterance "skyscrapers in Houston") - (targetFormula (and (fb:type.object.type fb:architecture.skyscraper) (fb:location.location.containedby fb:en.houston))) -) - -(example - (utterance "attraction of Rajahmundry") - (targetFormula (and (fb:type.object.type fb:travel.tourist_attraction) (fb:location.location.containedby fb:en.rajahmundry))) -) - -(example - (utterance "wife of Abraham") - (targetFormula (and (fb:people.person.gender fb:en.female) (fb:people.person.spouse_s (fb:people.marriage.spouse fb:en.abraham)))) -) - -(example - (utterance "native of Indianapolis , Indiana") - (targetFormula (fb:people.person.place_of_birth fb:en.indianapolis)) -) - -(example - (utterance "member of Barcelona") - (targetFormula (fb:soccer.football_player.current_team (fb:soccer.football_roster_position.team fb:en.fc_barcelona))) -) - -(example - (utterance "city of Nepal") - (targetFormula (and (fb:type.object.type fb:location.citytown) (fb:location.location.containedby fb:en.nepal))) -) - -(example - (utterance "home of ConocoPhillips in Canada") - (targetFormula (and (fb:location.location.containedby fb:en.canada) (!fb:location.mailing_address.citytown (!fb:organization.organization.headquarters fb:en.conocophillips_canada_ltd)))) -) - diff --git a/freebase/data/jonathan_tutorial.grammar b/freebase/data/jonathan_tutorial.grammar deleted file mode 100644 index 9466800..0000000 --- a/freebase/data/jonathan_tutorial.grammar +++ /dev/null @@ -1,50 +0,0 @@ -# Jonathan Berant's solution to the tutorial. - -#lexicon -(rule $Binary (area) (ConstantFn fb:location.location.area (-> fb:type.any fb:type.any))) -(rule $Unary (city) (ConstantFn (fb:type.object.type fb:location.citytown) fb:location.citytown)) -(rule $Unary (cities) (ConstantFn (fb:type.object.type fb:location.citytown) fb:location.citytown)) -(rule $Superlative (with the largest) (ConstantFn (lambda x (lambda y (argmax 1 1 (var y) (var x)))))) -(rule $TOP5 (top 5) (ConstantFn (lambda x (lambda y (argmax 1 5 (var x) (var y)))))) -(rule $RANK2 (second) (ConstantFn (lambda y (lambda x (argmax 2 1 (var x) (var y)))) (-> (-> fb:type.any fb:type.any) (-> fb:type.any fb:type.any)))) -(rule $Unary (countries) (ConstantFn (fb:type.object.type fb:location.country) fb:location.country)) -(rule $Unary (country) (ConstantFn (fb:type.object.type fb:location.country) fb:location.country)) -(rule $Unit (squared kilometers) (IdentityFn)) -(rule $Larger (at least) (ConstantFn (lambda x (>= (var x))))) -(rule $Binary (capital) (ConstantFn fb:location.country.capital (-> fb:type.any fb:type.any))) -(rule $Unary (states) (ConstantFn (fb:type.object.type fb:location.us_state) fb:location.us_state)) -(rule $Binary (bordering) (ConstantFn (lambda x (fb:location.location.adjoin_s (fb:location.adjoining_relationship.adjoins (var x)))) (-> fb:type.any fb:type.any))) -(rule $Entity (oregon) (ConstantFn fb:en.oregon fb:type.any)) -(rule $Entity (washington) (ConstantFn fb:en.washington fb:type.any)) -(rule $Conjunction (and) (ConstantFn (lambda x (lambda y (lambda z (and ((var z) (var x)) ((var z) (var y)))))) (-> fb:type.any (-> fb:type.any (-> (-> fb:type.any fb:type.any) fb:type.any))))) -(rule $Binary (tallest) (ConstantFn fb:geography.mountain.elevation (-> fb:type.any fb:type.any))) -(rule $Unary (mountain) (ConstantFn (fb:type.object.type fb:geography.mountain) fb:geography.mountain)) -(rule $Binary (in) (ConstantFn fb:location.location.containedby (-> fb:type.any fb:type.any))) -(rule $Unary (france) (ConstantFn fb:en.france fb:type.any)) -#question 1 -(rule $SuperBinary ($Superlative $Binary) (JoinFn forward betaReduce)) -(rule $SuperBinaryUnary ($Unary $SuperBinary) (JoinFn backward betaReduce)) -(rule $ROOT ($SuperBinaryUnary) (IdentityFn)) -#question 2 -(rule $Number ($TOKEN) (NumberFn)) -(rule $SuperUnary ($TOP5 $Unary) (JoinFn forward betaReduce)) -(rule $SuperBinaryUnary ($SuperUnary by $Binary) (JoinFn forward betaReduce)) -#question 3 -(rule $LargerNumber ($Larger $Number) (JoinFn forward betaReduce)) -(rule $LargerNumber ($LargerNumber $Unit) (IdentityFn)) -(rule $Set ($Binary $LargerNumber) (JoinFn forward)) -(rule $Set ($Unary whose $Set) (MergeFn and)) -#question 4 -(rule $ConjunctionEntity ($Conjunction $Entity) (JoinFn forward betaReduce)) -(rule $ConjunctionEntityEntity ($Entity $ConjunctionEntity) (JoinFn backward betaReduce)) -(rule $Set ($Binary $ConjunctionEntityEntity) (JoinFn backward betaReduce)) -#question 5 -(rule $Superlative ($RANK2) (IdentityFn)) -(rule $SuperBinaryUnary ($SuperBinary $Set) (JoinFn forward betaReduce)) -#question 6 - skipping this - -#General -(rule $Set ($Binary $Set) (JoinFn forward)) -(rule $Set ($Unary) (IdentityFn)) -(rule $Set ($Unary $Set) (MergeFn and)) -(rule $ROOT ($Set) (IdentityFn)) diff --git a/freebase/data/jonathan_type_raise.grammar b/freebase/data/jonathan_type_raise.grammar deleted file mode 100644 index d4e83d6..0000000 --- a/freebase/data/jonathan_type_raise.grammar +++ /dev/null @@ -1,18 +0,0 @@ -#lexicon -(rule $Binary (bordering) (ConstantFn (lambda x (fb:location.location.adjoin_s (fb:location.adjoining_relationship.adjoins (var x)))) (-> fb:type.any fb:type.any))) -(rule $Entity (oregon) (ConstantFn fb:en.oregon fb:type.any)) -(rule $Entity (nevada) (ConstantFn fb:en.nevada fb:type.any)) -(rule $Entity (arizona) (ConstantFn fb:en.arizona fb:type.any)) -#type raising -(rule $FunctionSet ($Entity) (JoinFn betaReduce forward (arg0 (lambda x (lambda f ((var f) (var x)))) (-> fb:type.any (-> (-> fb:type.any fb:type.any) fb:type.any))))) - -(rule $Conj (and) (ConstantFn (lambda a (lambda b (lambda g (and ((var a) (var g)) ((var b) (var g)))))) (-> (-> (-> fb:type.any fb:type.any) fb:type.any) (-> (-> (-> fb:type.any fb:type.any) fb:type.any) (-> (-> fb:type.any fb:type.any) fb:type.any))))) - -(rule $ConjRight ($Conj $FunctionSet) (JoinFn betaReduce forward)) -(rule $FunctionSet ($FunctionSet $ConjRight) (JoinFn betaReduce backward)) - -#General -(rule $Set ($Binary $FunctionSet) (JoinFn betaReduce backward)) -#(rule $Set ($Binary $Set) (JoinFn betaReduce forward)) -#(rule $Set ($Entity) (IdentityFn)) -(rule $ROOT ($Set) (IdentityFn)) diff --git a/freebase/data/obama.examples b/freebase/data/obama.examples deleted file mode 100644 index b1aea8b..0000000 --- a/freebase/data/obama.examples +++ /dev/null @@ -1,4 +0,0 @@ -(example - (utterance "where was obama born") - (targetValue (list (name fb:en.honolulu))) -) diff --git a/freebase/scripts/create-unary-and-binary-lexicons b/freebase/scripts/create-unary-and-binary-lexicons deleted file mode 100755 index e5fec56..0000000 --- a/freebase/scripts/create-unary-and-binary-lexicons +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Creating unary and binary lexicons given five arguments -# (1) Freebase file -# (2) Freebase schema file -# (3) linked unary open IE file -# (4) linked binary open IE file -# (5) work directory where all files are stored - -FREEBASE_FILE=$1 -SCHEMA_FILE=$2 -LINKED_UNARY_FILE=$3 -LINKED_BINARY_FILE=$4 -WORK_DIR=$5 - -echo "Creating working directory" -mkdir -p $WORK_DIR -./scripts/generate-files-for-lexicon.sh $FREEBASE_FILE $SCHEMA_FILE $WORK_DIR -echo "Generating unary lexicon..." -./scripts/lexicon @task=align_unary @linkedUnaryFile=$LINKED_UNARY_FILE @freebaseFile=$FREEBASE_FILE @workDir=$WORK_DIR -echo "Generating binary lexicon..." -./scripts/lexicon @task=align_binary @linkedBinaryFile=$LINKED_BINARY_FILE @freebaseFile=$FREEBASE_FILE @workDir=$WORK_DIR diff --git a/freebase/scripts/index-entities.sh b/freebase/scripts/index-entities.sh deleted file mode 100755 index 0a9ed7c..0000000 --- a/freebase/scripts/index-entities.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# arg0 - entity info file (/user/jobernat/scr/fb_data/aug_13/entityInfo.txt -# arg1 - target dir (/u/nlp/data/semparse/lucene/4.4/inexact -# arg2 - indexing strategy (inexact) - -# TODO: document; move into lexicon -usage() { - echo "Usage: $0 (entityFile) (indexDir) (indexStrategy)" - exit 1 -} - -if [ $# -lt 3 ]; then - usage -fi -java -Xmx3g -cp "classes:lib/*" edu.stanford.nlp.sempre.fbalignment.index.FbEntityIndexer $1 $2 $3 diff --git a/freebase/scripts/lookup-freebase.rb b/freebase/scripts/lookup-freebase.rb deleted file mode 100755 index 4e08563..0000000 --- a/freebase/scripts/lookup-freebase.rb +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/ruby - -require 'uri' -require 'json' - -# Query freebase.com for the mid of an entity (e.g., fb:en.barack_obama) - -def slash(id) - reverse = false - if id =~ /^!(.+)$/ - reverse = true - id = $1 - end - id =~ /^fb:(.+)$/ or raise "Bad: #{id}" - (reverse ? '!' : '') + '/' + $1.gsub(/\./, '/') -end - -def unslash(slash_id) - slash_id =~ /^\/(.+)$/ or raise "Bad: #{slash_id}" - 'fb:' + $1.gsub(/\//, '.') -end - -def process(formula) - # (!fb:broadcast.tv_station_owner.tv_stations fb:en.nbc) - if formula =~ /^\((!?fb:.+) (fb:.+)\)$/ - property = slash($1) - entity = slash($2) - query = "[{ \"id\": null, \"mid\": null, \"name\": null, \"#{property}\": { \"id\": \"#{entity}\" } }]" - elsif formula =~ /\(\(lambda x \((!?fb:.+) \((!?fb:.+) \(var x\)\)\)\) (fb:.+)\)$/ - property1 = slash($1) - property2 = slash($2) - entity = slash($3) - query = "[{ \"id\": null, \"mid\": null, \"name\": null, \"#{property1}\": { \"#{property2}\": { \"id\": \"#{entity}\" } } }]" - elsif formula =~ /^(fb:.+)$/ - entity = slash($1) - query = "[{ \"id\": \"#{entity}\", \"mid\": null, \"name\": null }]" - else - raise "Bad: #{formula}" - end - url = 'https://www.googleapis.com/freebase/v1/mqlread/?query=' + URI::escape(query).gsub(/\[/, '%5B').gsub(/\]/, '%5D') - - $stderr.puts query - map = JSON::parse(`curl -s '#{url}'`) - map['result'].each { |ent| - mid = unslash(ent['mid']) - id = unslash(ent['id']) - canonical_id = `grep "^#{mid} " /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map`.split[1] - puts [mid, id, canonical_id, ent['name']].join("\t") - } -end - -if ARGV.size == 0 - while true - print "formula> " - formula = gets - if not formula - puts - break - end - process(formula) - - system "./run @mode=query @index=jackson:3090 -formula '#{formula}'" - end -else - ARGV.each { |formula| process(formula) } -end diff --git a/freebase/scripts/preprocess-freebase b/freebase/scripts/preprocess-freebase deleted file mode 100755 index cd28f2a..0000000 --- a/freebase/scripts/preprocess-freebase +++ /dev/null @@ -1,33 +0,0 @@ -# These are the steps to take the raw Freebase dump, canonicalize ids, build a -# SPARQL index, convert examples to use the canonical ids. -# NOTE: do not run this script because it will overwrite a lot of things. - -exit 1 - -# 1) Build a map from ids to canonical ids (based on the raw Freebase dump). 8GB -java -cp classes:lib/* edu.stanford.nlp.sempre.freebase.BuildCanonicalIdMap -rawPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.gz -canonicalIdMapPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map - -# 2a) Replace ids with canonical ids (and do minor filtering) in the Freebase graph. -java -cp classes:lib/* edu.stanford.nlp.sempre.freebase.CanonicalizeIds -canonicalIdMapPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map -rawPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.gz -canonicalizedPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized - -# 2b) Replace ids with canonical ids in the examples files. -java -cp classes:lib/* edu.stanford.nlp.sempre.CanonicalizeExamples -canonicalIdMapPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonical-id-map -examplePaths data/yates.examples /u/nlp/data/semparse/google/dataset_8/bfs_8_good_checked.examples3 - -# 3) Filter the Freebase graph using properties that occur in examples files and Tom Lin alignments. -fig/bin/qcreate -statePath /u/nlp/data/semparse/rdf/scr/`hostname | cut -f 1 -d .`/state java -cp classes:lib/* edu.stanford.nlp.sempre.freebase.FilterFreebase \ - -examplesPath data/yates.examples.canonicalized /u/nlp/data/semparse/google/dataset_8/bfs_8_good_checked.examples3.canonicalized \ - -keepProperties <(awk '{print $2}' lib/binaryInfoStringMatch.txt | grep ^fb:) \ - -inPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized -execDir _OUTPATH_ -overwriteExecDir - -# ...or keep everything -time fig/bin/qcreate -statePath /u/nlp/data/semparse/rdf/scr/`hostname | cut -f 1 -d .`/state java -cp classes:lib/* edu.stanford.nlp.sempre.freebase.FilterFreebase -inPath /u/nlp/data/semparse/scr/freebase/freebase-rdf-2013-06-09-00-00.canonicalized -keepAllProperties -execDir _OUTPATH_ -overwriteExecDir - -# 4) Index the appropriate subset of the Freebase graph. -./run @mode=indexfreebase @exec=90 @backend=virtuoso - -# 5) Extract a schema file -scripts/extract-freebase-schema.rb http://jackson:3090/sparql /u/nlp/data/semparse/rdf/scr/`hostname | cut -f 1 -d .`/state/execs/90.exec/schema-2.ttl - -# 6) Execute formulas in examples files to check if everything is okay. -java -cp classes:lib/* edu.stanford.nlp.sempre.ExecuteExamples -examplesPath data/yates.examples.canonicalized /u/nlp/data/semparse/google/dataset_8/bfs_8_good_checked.examples3.canonicalized -SparqlExecutor.endpointUrl http://jackson:3090/sparql | tee log -java -cp classes:lib/* edu.stanford.nlp.sempre.ExecuteExamples -examplesPath data/yates.examples.canonicalized -SparqlExecutor.endpointUrl http://jackson:3090/sparql | tee log diff --git a/pull-dependencies b/pull-dependencies index 5659ca2..b69420d 100755 --- a/pull-dependencies +++ b/pull-dependencies @@ -126,7 +126,6 @@ addModule('corenlp', 'Stanford CoreNLP 3.2.0 (for backward reproducibility)', la 'jollyday.jar' => 'jollyday.jar'}.each { |key, value| system "ln -sfv stanford-corenlp-full-2013-06-20/#{key} lib/#{value}" or exit 1 } - pull('/u/nlp/data/semparse/resources/stanford-corenlp-caseless-2013-06-07-models.jar', '', {:symlink => true}) }) addModule('freebase', 'Freebase: need to construct Freebase schemas', lambda { diff --git a/src/edu/stanford/nlp/sempre/Learner.java b/src/edu/stanford/nlp/sempre/Learner.java index be023b7..2b9f92a 100644 --- a/src/edu/stanford/nlp/sempre/Learner.java +++ b/src/edu/stanford/nlp/sempre/Learner.java @@ -108,9 +108,8 @@ public class Learner { for (String group : dataset.groups()) { boolean lastIter = (iter == numIters); boolean updateWeights = opts.updateWeights && group.equals("train") && !lastIter; // Don't train on last iteration - if (parser instanceof MutatingParser) { - ((MutatingParser) parser).mutate(iter, numIters, group); - } + // Allow the parser to change behavior based on current group and iteration + parser.mutate(iter, numIters, group); Evaluation eval = processExamples( iter, group, diff --git a/src/edu/stanford/nlp/sempre/MutatingParser.java b/src/edu/stanford/nlp/sempre/MutatingParser.java deleted file mode 100644 index 45389aa..0000000 --- a/src/edu/stanford/nlp/sempre/MutatingParser.java +++ /dev/null @@ -1,12 +0,0 @@ -package edu.stanford.nlp.sempre; - -/** - * Parser that can mutate after each training iteration. - * - * @author ppasupat - */ -public interface MutatingParser { - - // This will be called after each training iteration. - public void mutate(int iter, int numIters, String group); -} diff --git a/src/edu/stanford/nlp/sempre/Parser.java b/src/edu/stanford/nlp/sempre/Parser.java index 7b28ae0..036a29e 100644 --- a/src/edu/stanford/nlp/sempre/Parser.java +++ b/src/edu/stanford/nlp/sempre/Parser.java @@ -137,8 +137,18 @@ public abstract class Parser { done.put(node, true); } + /** + * Override this method to change the parser's behavior based on current + * group name and iteration number. This method will be called at the + * beginning of each data group. + */ + public void mutate(int iter, int numIters, String group) { + // DEFAULT: Do nothing. + } + // Main thing for parsers to implement. public abstract ParserState newParserState(Params params, Example ex, boolean computeExpectedCounts); + public Params getSearchParams(Params params) { return params; } /** diff --git a/src/edu/stanford/nlp/sempre/SemanticFn.java b/src/edu/stanford/nlp/sempre/SemanticFn.java index db5a0ff..9d94249 100644 --- a/src/edu/stanford/nlp/sempre/SemanticFn.java +++ b/src/edu/stanford/nlp/sempre/SemanticFn.java @@ -36,12 +36,6 @@ public abstract class SemanticFn { this.tree = tree; } - public SemanticFn copy() { - SemanticFn fn = (SemanticFn) Utils.newInstanceHard(this.getClass().getCanonicalName()); - fn.init(tree); - return fn; - } - public interface Callable { String getCat(); int getStart();