mirror of https://github.com/percyliang/sempre
78 lines
2.2 KiB
Ruby
Executable File
78 lines
2.2 KiB
Ruby
Executable File
#!/usr/bin/ruby
|
|
|
|
require 'pathname'
|
|
require File.dirname(Pathname.new($0).realpath) + '/../lib/myutils'
|
|
|
|
# Print out a chunk of a file. Specifically, print out the header lines.
|
|
$file, $headerNumLines, $chunkSize, $indices, $printNumChunks, $verbose = extractArgs(:spec => [
|
|
['file', String, nil, true, 'Big file to read'],
|
|
['headerNumLines', Fixnum, 0, false, 'Number of header lines to include'],
|
|
['chunkSize', String, '100M', false, 'Size of a chunk (e.g., 1024, 1K, 1M, 1G)'],
|
|
['indices', [Fixnum], [], false, 'Which chunk(s) we want'],
|
|
['printNumChunks', TrueClass, false, false, 'Print out number of chunks'],
|
|
['verbose', Fixnum, 0, false, 'Verbosity level (to stderr)'],
|
|
nil])
|
|
|
|
f = open($file, 'r')
|
|
header = (0...$headerNumLines).map { f.gets }
|
|
|
|
startPos = f.tell
|
|
f.seek(0, IO::SEEK_END)
|
|
endPos = f.tell
|
|
|
|
def parseSize(s)
|
|
return Integer(Float($1) * 1024**1) if s =~ /^(.+)[kK]$/
|
|
return Integer(Float($1) * 1024**2) if s =~ /^(.+)[mM]$/
|
|
return Integer(Float($1) * 1024**3) if s =~ /^(.+)[gG]$/
|
|
return Integer(s)
|
|
end
|
|
|
|
$chunkSize = parseSize($chunkSize)
|
|
|
|
totalSize = endPos - startPos
|
|
numChunks = (totalSize + $chunkSize - 1) / $chunkSize
|
|
$stderr.puts "start = #{startPos}, end = #{endPos}, chunkSize = #{$chunkSize}, totalSize = #{totalSize}, #{numChunks} chunks" if $verbose >= 1
|
|
puts numChunks if $printNumChunks
|
|
|
|
# Return the first position of the i-th chunk which is a new line.
|
|
# Seek to that point.
|
|
findBeginOfLine = lambda { |i|
|
|
if i == 0
|
|
f.seek(startPos, IO::SEEK_SET)
|
|
return startPos
|
|
else
|
|
pos = [endPos, startPos + i * $chunkSize - 1].min
|
|
f.seek(pos, IO::SEEK_SET)
|
|
while true
|
|
c = f.read(1)
|
|
#p [f.tell-1, c]
|
|
break if c == nil || c == "\n"
|
|
end
|
|
return f.tell
|
|
end
|
|
}
|
|
|
|
dump = lambda { |pos0,pos1|
|
|
f.seek(pos0, IO::SEEK_SET)
|
|
pos = pos0
|
|
while true
|
|
n = [16384, pos1-pos].min
|
|
break if n == 0
|
|
print f.read(n)
|
|
pos += n
|
|
end
|
|
}
|
|
|
|
printedHeader = false
|
|
$indices.each { |i|
|
|
pos0 = findBeginOfLine.call(i)
|
|
pos1 = findBeginOfLine.call(i+1)
|
|
next if pos0 == pos1
|
|
$stderr.puts "Chunk #{i}/#{numChunks}: #{pos0} to #{pos1}" if $verbose >= 1
|
|
if not printedHeader
|
|
printedHeader = true
|
|
header.each { |line| puts line }
|
|
end
|
|
dump.call(pos0, pos1)
|
|
}
|