mirror of https://github.com/percyliang/sempre
234 lines
8.5 KiB
Ruby
Executable File
234 lines
8.5 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
|
|
# This is the main entry point for running SHRDLURN. See
|
|
# fig/lib/execrunner.rb for more documentation for how commands are generated.
|
|
# There are a bunch of modes that this script can be invoked with, which
|
|
# loosely correspond to the modules.
|
|
|
|
$: << 'fig/lib'
|
|
require 'execrunner'
|
|
$optPrefix = '-'
|
|
$path = 'interactive'
|
|
$output = 'interactive/output'
|
|
$modes = []
|
|
def addMode(name, description, func)
|
|
$modes << [name, description, func]
|
|
end
|
|
|
|
def codalab(dependencies=nil)
|
|
# Set @cl=1 to run job on CodaLab
|
|
dependencies ||= l(':fig', ':lib', ':module-classes.txt', ':libsempre')
|
|
l(
|
|
letDefault(:cl, 0),
|
|
sel(:cl,
|
|
l(),
|
|
l('cl', 'run', dependencies, '---', 'LC_ALL=C.UTF-8'),
|
|
nil),
|
|
nil)
|
|
end
|
|
|
|
def header(modules='core', codalabDependencies=nil)
|
|
l(
|
|
codalab(codalabDependencies),
|
|
# Queuing system
|
|
letDefault(:q, 0), sel(:q, l(), l('fig/bin/q', '-shareWorkingPath', o('mem', '5g'), o('memGrace', 10), '-add', '---')),
|
|
# Create execution directory
|
|
letDefault(:pooldir, 1),
|
|
sel(:pooldir, l(), 'fig/bin/qcreate'),
|
|
# Run the Java command...
|
|
'java',
|
|
'-ea',
|
|
'-Dmodules='+modules,
|
|
# Memory size
|
|
letDefault(:memsize, 'default'),
|
|
sel(:memsize, {
|
|
'tiny' => l('-Xms2G', '-Xmx4G'),
|
|
'low' => l('-Xms5G', '-Xmx7G'),
|
|
'default' => l('-Xms8G', '-Xmx10G'),
|
|
'medium' => l('-Xms12G', '-Xmx14G'),
|
|
'high' => l('-Xms20G', '-Xmx24G'),
|
|
'higher' => l('-Xms40G', '-Xmx50G'),
|
|
'impressive' => l('-Xms75G', '-Xmx90G'),
|
|
}),
|
|
# Classpath
|
|
'-cp', 'libsempre/*:lib/*',
|
|
# Profiling
|
|
letDefault(:prof, 0), sel(:prof, l(), '-Xrunhprof:cpu=samples,depth=100,file=_OUTPATH_/java.hprof.txt'),
|
|
# Debugging
|
|
letDefault(:debug, 0), sel(:debug, l(), l('-Xdebug', '-Xrunjdwp:server=y,transport=dt_socket,suspend=y,address=8898')),
|
|
nil)
|
|
end
|
|
|
|
def figOpts; l(selo(:pooldir, 'execDir', 'exec', '_OUTPATH_'), o('overwriteExecDir'), o('addToView', 0)) end
|
|
|
|
############################################################
|
|
# Unit tests
|
|
|
|
addMode('backup', 'small commands like run community server, backup, or simulator', lambda { |e| l(
|
|
lambda { |e| system 'echo "backing up with mv"'},
|
|
lambda { |e| system 'mkdir -p ./int-backup/'},
|
|
letDefault(:msg, 'backing up, no message'),
|
|
lambda { |e| l('echo ', :msg, '> ./int-backup/message')},
|
|
lambda { |e| l('echo ', '`date +%Y-%m-%d.%H:%M:%S`', '>> ./int-backup/message')},
|
|
lambda { |e| system 'mv int-output int-backup/`date +%Y-%m-%d.%H:%M:%S`'},
|
|
lambda { |e| system 'mkdir -p ./int-output'},
|
|
nil)})
|
|
|
|
addMode('backup-data', 'put community-server into trash with time stamp', lambda { |e| l(
|
|
lambda { |e| system 'echo "backing up data with mv"'},
|
|
lambda { |e| system 'mkdir -p ./community-server/data-backup'},
|
|
lambda { |e| system 'mv ./community-server/data ./community-server/data-backup/`date +%Y-%m-%d.%H:%M:%S`'},
|
|
nil)})
|
|
|
|
addMode('trash', 'put int-output into trash with time stamp', lambda { |e| l(
|
|
lambda { |e| system 'echo "trashing int-output with time stamp"'},
|
|
lambda { |e| system 'mv int-output int-output-trash-`date +%Y-%m-%d.%H:%M:%S`'},
|
|
lambda { |e| system 'rm -rf int-output-trash-*'},
|
|
lambda { |e| system 'mkdir -p ./int-output'},
|
|
nil)})
|
|
|
|
addMode('test', 'Run unit tests for interactive stuff', lambda { |e|
|
|
l(
|
|
'java', '-ea', '-Xmx12g', '-cp', 'libsempre/*:lib/*',
|
|
letDefault(:debug, 0), sel(:debug, l(), l('-Xdebug', '-Xrunjdwp:server=y,transport=dt_socket,suspend=y,address=8898')),
|
|
'org.testng.TestNG',
|
|
lambda { |e|
|
|
if e[:class]
|
|
l('-testclass', 'edu.stanford.nlp.sempre.interactive.test.' + e[:class])
|
|
else
|
|
"./#{$path}/testng.xml"
|
|
end
|
|
},
|
|
nil)
|
|
})
|
|
|
|
addMode('simulator', 'run the simulator', lambda { |e| l(
|
|
# rlwrap,
|
|
header('core,interactive'),
|
|
'edu.stanford.nlp.sempre.interactive.Simulator',
|
|
figOpts,
|
|
letDefault(:server, 'local'),
|
|
sel(:server, {
|
|
'local' => o('serverURL', 'http://localhost:8410'),
|
|
'remote' => o('serverURL', 'http://jonsson.stanford.edu:8410')
|
|
}),
|
|
# set to 0 to enable logging
|
|
o('numThreads', 1),
|
|
letDefault(:sandbox, 'full'),
|
|
sel(:sandbox, {
|
|
'all' => o('reqParams', 'grammar=0\&cite=0\&learn=0\&logging=0'),
|
|
'nolog' => o('reqParams', 'grammar=0\&cite=0\&learn=1\&logging=0'),
|
|
'nolearn' => o('reqParams', 'grammar=1\&cite=1\&learn=0\&logging=0'),
|
|
'none' => o('reqParams', 'grammar=1\&cite=1\&learn=1\&logging=0'),
|
|
'nocite' => o('reqParams', 'grammar=1\&cite=0\&learn=1\&logging=0'),
|
|
}),
|
|
letDefault(:task, 'sidaw'),
|
|
sel(:task, {
|
|
'freebuild' => o('logFiles', "./#{$path}/queries/freebuild.json.gz"),
|
|
'freebuilddef' => o('logFiles', "./#{$path}/queries/freebuild.def.json.gz"),
|
|
|
|
'qual1' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier1-0118.json.gz"),
|
|
'qual2' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier2-0129.json.gz"),
|
|
'qual3' => o('logFiles', "./#{$path}/queries/rawqueries/qualifier3-0201.json.gz"), # both 2 and 3
|
|
'free1' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild1-0121.json.gz"),
|
|
'free2' => o('logFiles', "./#{$path}/queries/rawqueries/freebuild2-0127.json.gz"),
|
|
}),
|
|
nil)})
|
|
|
|
############################################################
|
|
# {2016-07-02} [sidaw]: interactive semantic parsing
|
|
addMode('voxelurn', 'interactive semantic parsing in a VoxelWorld', lambda { |e| l(
|
|
#rlwrap,
|
|
header('core,interactive'),
|
|
'edu.stanford.nlp.sempre.Main',
|
|
#figOpts,
|
|
o('server'),
|
|
o('masterType', 'edu.stanford.nlp.sempre.interactive.InteractiveMaster'),
|
|
o('Executor', 'interactive.DALExecutor'),
|
|
o('LanguageAnalyzer', 'interactive.DALAnalyzer'),
|
|
o('DALExecutor.convertNumberValues', true),
|
|
o('DALExecutor.printStackTrace', true),
|
|
o('VoxelWorld.maxBlocks', 100000),
|
|
selo(0, 'DALExecutor.worldType', 'VoxelWorld', 'CalendarWorld', 'Otherworld'),
|
|
selo(0, 'Grammar.inPaths', "./#{$path}/voxelurn.grammar"),
|
|
|
|
o('Params.initWeightsRandomly', false),
|
|
o('Grammar.binarizeRules', false),
|
|
o('Grammar.useApplyFn', 'interactive.ApplyFn'),
|
|
|
|
o('LanguageAnalyzer.lowerCaseTokens', true),
|
|
|
|
o('Parser.pruneErrorValues', true),
|
|
o('Parser', 'interactive.InteractiveBeamParser'),
|
|
o('Parser.callSetEvaluation', false),
|
|
o('Parser.coarsePrune', true),
|
|
|
|
o('Parser.beamSize', 50),
|
|
o('InteractiveBeamParser.maxNewTreesPerSpan', 5001),
|
|
o('ParserState.customExpectedCounts', 'None'),
|
|
|
|
selo(0, 'InteractiveBeamParser.floatStrategy', 'Never', 'NoParse', 'Always'),
|
|
o('InteractiveBeamParser.trackedCats', 'Number', 'Numbers', 'Color', 'Direction', 'Set', 'Sets', 'Action', 'Actions'),
|
|
|
|
o('Derivation.derivComparator', 'AnchorPriorityScoreComparator'),
|
|
o('Params.l1Reg', 'nonlazy'),
|
|
o('Params.l1RegCoeff', 0.0001),
|
|
|
|
o('Params.initStepSize', 0.1),
|
|
o('Params.adaptiveStepSize', true),
|
|
#o('Params.stepSizeReduction', 0.25),
|
|
|
|
o('FeatureExtractor.featureComputers', 'interactive.DALFeatureComputer'),
|
|
o('FeatureExtractor.featureDomains', ':rule', ':span', ':stats', ':scope', ':social', ':window'),
|
|
# o('FeatureExtractor.featureDomains', ':rule'),
|
|
|
|
o('InteractiveMaster.intOutputPath', './int-output/'),
|
|
o('InteractiveMaster.onlyInteractive', true),
|
|
o('InteractiveUtils.citationPath', './int-output/citation/'),
|
|
|
|
o('InteractiveMaster.useAligner', false),
|
|
o('InteractiveMaster.maxSequence', 20),
|
|
o('InteractiveMaster.maxChars', 200),
|
|
|
|
o('DefinitionAligner.strategies', 'ExactExclusion'),
|
|
|
|
o('InteractiveServer.numThreads', 16),
|
|
o('InteractiveServer.maxCandidates', 50),
|
|
o('InteractiveServer.queryLogPath', './int-output/query.log'),
|
|
o('InteractiveServer.responseLogPath', './int-output/response.log'),
|
|
o('InteractiveServer.port', 8410),
|
|
|
|
o('GrammarInducer.useBestPacking', true),
|
|
o('GrammarInducer.useSimplePacking', true),
|
|
o('GrammarInducer.maxNonterminals', 3),
|
|
|
|
o('Derivation.showTypes', false),
|
|
o('Derivation.showValues', false),
|
|
o('Derivation.showRules', false),
|
|
o('Derivation.anchoredBonus', 1.0),
|
|
|
|
o('NumberFn.allowedRange', 0, 100),
|
|
o('SimpleLexicon.inPaths', "./#{$path}/csscolors.lexicon"),
|
|
|
|
lambda { |e| system 'mkdir -p ./int-output/'; nil},
|
|
lambda { |e| system 'mkdir -p ./int-output/log/'; nil},
|
|
lambda { |e| system 'mkdir -p ./int-output/citation/'; nil},
|
|
nil) })
|
|
############################################################
|
|
|
|
if ARGV.size == 0
|
|
puts "#{$0} @mode=<mode> [options]"
|
|
puts
|
|
puts 'This is the main entry point for all interactive related modes.'
|
|
puts "Modes:"
|
|
$modes.each { |name,description,func|
|
|
puts " #{name}: #{description}"
|
|
}
|
|
end
|
|
|
|
modesMap = {}
|
|
$modes.each { |name,description,func|
|
|
modesMap[name] = func
|
|
}
|
|
run!(sel(:mode, modesMap))
|