sempre/parasempre

#!/usr/bin/ruby

$: << 'fig/lib'
require 'execrunner'

# Note: run this on the NLP machines because SPARQL server is running on jack.
if ARGV.size == 0
  puts "Usage:"
  puts "  ./parasempre @mode=train @domain=<webquestions|free917> @sparqlserver=<host:port> @cacheserver=<none|local>"
  puts "Additional options can be any of the following:"
  puts "  - Additional program options (e.g., -BeamParser.beamSize 3)"
  puts "  - Execrunner options, which select the options to include (@data=0)"
  exit 1
end

system "mkdir -p trans_state/execs"
system "touch trans_state/lastExec"

def header
  l(
    letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
    'fig/bin/qcreate',
    '-statePath',
    'trans_state',
    (File.exists?('/u/nlp/bin/java7') ? '/u/nlp/bin/java7' : 'java'),
    '-ea',
    '-Xmx10g',
    '-cp', 'classes:'+Dir['lib/*.jar'].join(':'),
    letDefault(:prof, 0), sel(:prof, 
                              l(), 
                              '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt',
                              '-Xrunhprof:heap=sites,file=_OUTPATH_/java.hprof.txt'),
  nil)
end

def sparqlOpts
  l(
    required(:sparqlserver, 'host:port of the Sparql server'),
    o('SparqlExecutor.endpointUrl', lambda{|e| 'http://'+e[:sparqlserver]+'/sparql'}),
  nil)
end

def defaultOpts
  l(
    o('execDir', '_OUTPATH_'), o('overwriteExecDir'),
    o('addToView', 0),
  nil)
end

def cachePaths(lexiconFnCachePath, sparqlExecutorCachePath)
  l(
    required(:cacheserver, 'none (don\'t cache to disk), local (write to local file), or <hostname>:<port> (hit the cacheserver)'),
    lambda { |e|
      cacheserver = e[:cacheserver]
      case cacheserver
      when 'none' then l()
      when 'local' then l( # Use files directly - don't run more than one job that does this!
        o('Lexicon.cachePath', 'LexiconFn.cache'),
        o('SparqlExecutor.cachePath', 'SparqlExecutor.cache'),
      nil)
      else l(
        o('Lexicon.cachePath', cacheserver + ':' + lexiconFnCachePath),
        o('SparqlExecutor.cachePath', cacheserver + ':' + sparqlExecutorCachePath),
      nil)
      end
    },
  nil)
end


$defaultFeatureDomains = [
  'Del',
  'Denotation',
  'Formula',
  'NamedEntity',
  #'Pt',
  'Subst',
  'WhType',
nil].compact

def unbalancedTrainDevSplit
    l(o('ParaphraseDataset.trainFrac', 0.8), o('ParaphraseDataset.devFrac', 0.2))
end

def allTrainSplit
    l(o('ParaphraseDataset.trainFrac', 1), o('ParaphraseDataset.devFrac', 0))
end

def train
  l(
    header,
    'edu.stanford.nlp.sempre.paraphrase.ParaphraseMain',
    defaultOpts, sparqlOpts,
    o('ParaphraseMain.mode','train'),
    required(:domain, 'domain (webquestions or free917)'), 
    selectDomain,
    o('Rulebase.ruleTypes','Pt','Syntax', 'Move', 'Subst'),
    o('ParaphraseFeatureMatcher.featureDomains', *$defaultFeatureDomains),
    o('Params.l1Reg','lazy'),
    o('Aligner.useWordnet',true),
    o('FeatureSimilarityComputer.mode','lexical_overlap'),
  nil)
end

def webquestions
  l(
    letDefault(:data, 0),
    sel(:data,
      l(o('ParaphraseDataset.parsingInPaths',
          'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json'),
          unbalancedTrainDevSplit,
       nil),
      l(o('ParaphraseDataset.parsingInPaths',
          'train,lib/data/webquestions/dataset_11/webquestions.examples.train.json',
          'test,lib/data/webquestions/dataset_11/webquestions.examples.test.json'),
          allTrainSplit,
      nil),
    nil),
    o('ParaphraseLearner.maxTrainIters',2),
    o('ParaphraseLearner.partialReward',true),
    o('ParaphraseParser.alignment',true),
    o('ParaphraseParser.vsm',true),
    o('Lexicon.entitySearchStrategy','inexact'),
    o('FormulaRetriever.supportCountUtterances',false),
    o('FormulaRetriever.filterRelations',true),
    o('FormulaRetriever.maxEntries',10),
    o('FormulaRetriever.conservativeEntityExtraction',true),
    o('Params.l1RegCoeff','0.00017782794'),
    o('VectorSpaceModel.wordVectorFile','lib/wordreprs/cbow-lowercase-50.vectors'),
    l(
      o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/inexact/'),
      cachePaths('LexiconFnWebQ.cache', 'SparqlExecutor.cache'),
    nil),
  nil)
end

def free917
  l(
    letDefault(:data, 0),
    sel(:data,
      l(o('ParaphraseDataset.parsingInPaths', 'train,data/free917.train.examples.canonicalized.json'),
        unbalancedTrainDevSplit,
       nil),
      l(o('ParaphraseDataset.parsingInPaths', 'train,data/free917.train.examples.canonicalized.json', 'test,data/free917.test.examples.canonicalized.json'),
        allTrainSplit,
      nil), 
    nil),
    o('ParaphraseLearner.maxTrainIters',10),
    o('ParaphraseLearner.partialReward',false),
    o('ParaphraseParser.alignment',true),
    o('ParaphraseParser.vsm',true),
    o('Lexicon.entitySearchStrategy','exact'),
    o('FormulaRetriever.supportCountUtterances',true),
    o('FormulaRetriever.filterRelations',false),
    o('FormulaRetriever.maxEntries',1000),
    o('FormulaRetriever.conservativeEntityExtraction',false),
    o('Params.l1RegCoeff',0.00562341325),
    o('VectorSpaceModel.wordVectorFile','lib/wordreprs/wordLexicon'),
    l(
      o('EntityLexicon.exactMatchIndex','lib/lucene/4.4/free917/'),
      cachePaths('LexiconFnFree917.cache', 'SparqlExecutor.cache'),
    nil),
  nil)
end

def selectDomain
  sel(:domain, {
    'webquestions' => webquestions,
    'free917' => free917,
  })
end


run!(
  sel(:mode, {
    'train' => train,
  }),
nil)