adding fig/bin/chunk

fig/bin/chunk

@@ -0,0 +1,77 @@
+#!/usr/bin/ruby
+
+require ENV['HOME'] + '/system/lib/ruby-myutils/myutils'
+
+# Print out a chunk of a file.  Specifically, print out the header lines.
+# Use 
+$file, $headerNumLines, $chunkSize, $indices, $printNumChunks, $verbose = extractArgs(:spec => [
+  ['file', String, nil, true, 'Big file to read'],
+  ['headerNumLines', Fixnum, 0, false, 'Number of header lines to include'],
+  ['chunkSize', String, '100M', false, 'Size of a chunk (e.g., 1024, 1K, 1M, 1G)'],
+  ['indices', [Fixnum], [], false, 'Which chunk(s) we want'],
+  ['printNumChunks', TrueClass, false, false, 'Print out number of chunks'],
+  ['verbose', Fixnum, 0, false, 'Verbosity level (to stderr)'],
+nil])
+
+f = open($file, 'r')
+header = (0...$headerNumLines).map { f.gets }
+
+startPos = f.tell
+f.seek(0, IO::SEEK_END)
+endPos = f.tell
+
+def parseSize(s)
+  return Integer(Float($1) * 1024**1) if s =~ /^(.+)[kK]$/
+  return Integer(Float($1) * 1024**2) if s =~ /^(.+)[mM]$/
+  return Integer(Float($1) * 1024**3) if s =~ /^(.+)[gG]$/
+  return Integer(s)
+end
+
+$chunkSize = parseSize($chunkSize)
+
+totalSize = endPos - startPos
+numChunks = (totalSize + $chunkSize - 1) / $chunkSize
+$stderr.puts "start = #{startPos}, end = #{endPos}, chunkSize = #{$chunkSize}, totalSize = #{totalSize}, #{numChunks} chunks" if $verbose >= 1
+puts numChunks if $printNumChunks
+
+# Return the first position of the i-th chunk which is a new line.
+# Seek to that point.
+findBeginOfLine = lambda { |i|
+  if i == 0
+    f.seek(startPos, IO::SEEK_SET)
+    return startPos
+  else
+    pos = [endPos, startPos + i * $chunkSize - 1].min
+    f.seek(pos, IO::SEEK_SET)
+    while true
+      c = f.read(1)
+      #p [f.tell-1, c]
+      break if c == nil || c == "\n"
+    end
+    return f.tell
+  end
+}
+
+dump = lambda { |pos0,pos1|
+  f.seek(pos0, IO::SEEK_SET)
+  pos = pos0
+  while true
+    n = [16384, pos1-pos].min
+    break if n == 0
+    print f.read(n)
+    pos += n
+  end
+}
+
+printedHeader = false
+$indices.each { |i|
+  pos0 = findBeginOfLine.call(i)
+  pos1 = findBeginOfLine.call(i+1)
+  next if pos0 == pos1
+  $stderr.puts "Chunk #{i}/#{numChunks}: #{pos0} to #{pos1}" if $verbose >= 1
+  if not printedHeader
+    printedHeader = true
+    header.each { |line| puts line }
+  end
+  dump.call(pos0, pos1)
+}

release-code.files

@@ -23,6 +23,7 @@ download-dependencies
 
 # Ruby utils: execrunner script etc.
 fig/bin/qcreate
+fig/bin/chunk
 fig/lib/execrunner.rb
 fig/lib/myutils.rb