Protobufs and common Java code for binary format.

build.sh

@@ -0,0 +1,2 @@
+protoc --java_out=generated.java src/osmformat.proto
+protoc --java_out=generated.java src/fileformat.proto

src.java/crosby/binary/BinaryParser.java

@@ -0,0 +1,109 @@
+package crosby.binary;
+
+
+import java.util.Date;
+import java.util.List;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+import crosby.binary.Osmformat;
+import crosby.binary.file.BlockReaderAdapter;
+import crosby.binary.file.FileBlock;
+import crosby.binary.file.FileBlockPosition;
+
+public abstract class BinaryParser implements BlockReaderAdapter {
+    private int granularity;
+    private long lat_offset;
+    private long lon_offset;
+    private int date_granularity;
+    private String strings[];
+
+    protected Date getDate(Osmformat.Info info) {
+      if (info.hasTimestamp()) {
+          return new Date(date_granularity * (long) info.getTimestamp());
+      } else
+          return NODATE;
+    }
+    public static final Date NODATE = new Date();
+
+    protected String getStringById(int id) {
+      return strings[id];
+    }
+    
+    @Override
+    public void handleBlock(FileBlock message) {
+        // TODO Auto-generated method stub
+        try {
+            if (message.getType().equals("OSMHeader")) {
+                Osmformat.HeaderBlock headerblock = Osmformat.HeaderBlock
+                        .parseFrom(message.getData());
+                parse(headerblock);
+            } else if (message.getType().equals("OSMData")) {
+                Osmformat.PrimitiveBlock primblock = Osmformat.PrimitiveBlock
+                        .parseFrom(message.getData());
+                parse(primblock);
+            }
+        } catch (InvalidProtocolBufferException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+            throw new Error("ParseError"); // TODO
+        }
+
+    }
+
+
+    @Override
+    public boolean skipBlock(FileBlockPosition block) {
+        // System.out.println("Seeing block of type: "+block.getType());
+        if (block.getType().equals("OSMData"))
+            return false;
+        if (block.getType().equals("OSMHeader"))
+            return false;
+        System.out.println("Skipped block of type: " + block.getType());
+        return true;
+    }
+    
+    
+    
+    public double parseLat(long degree) {
+      // Support non-zero offsets. (We don't currently generate them)
+      return (granularity * degree + lat_offset) * .000000001;
+    }
+
+    public double parseLon(long degree) {
+      // Support non-zero offsets. (We don't currently generate them)
+       return (granularity * degree + lon_offset) * .000000001;
+    }
+   
+    
+    public void parse(Osmformat.PrimitiveBlock block) {
+        Osmformat.StringTable stablemessage = block.getStringtable();
+        strings = new String[stablemessage.getSCount()];
+
+        for (int i = 0; i < strings.length; i++) {
+            strings[i] = stablemessage.getS(i).toStringUtf8();
+        }
+
+        granularity = block.getGranularity();
+        lat_offset = block.getLatOffset();
+        lon_offset = block.getLonOffset();
+        date_granularity = block.getDateGranularity();
+
+        for (Osmformat.PrimitiveGroup groupmessage : block
+                .getPrimitivegroupList()) {
+            // Exactly one of these should trigger on each loop.
+            parseNodes(groupmessage.getNodesList());
+            parseWays(groupmessage.getWaysList());
+            parseRelations(groupmessage.getRelationsList());
+            if (groupmessage.hasDense())
+                parseDense(groupmessage.getDense());
+        }
+    }
+    
+    protected abstract void parseRelations(List<Osmformat.Relation> rels);
+    protected abstract void parseDense(Osmformat.DenseNodes nodes);
+    protected abstract void parseNodes(List<Osmformat.Node> nodes);
+    protected abstract void parseWays(List<Osmformat.Way> ways);
+    protected abstract void parse(Osmformat.HeaderBlock header);
+
+}

src.java/crosby/binary/BinarySerializer.java

@@ -0,0 +1,145 @@
+package crosby.binary;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import crosby.binary.file.BlockOutputStream;
+import crosby.binary.file.FileBlock;
+
+/**
+ * Generic serializer common code
+ * 
+ * Serialize a set of blobs and process them. Subclasses implement handlers for
+ * different API's (osmosis, mkgmap, splitter, etc.)
+ * 
+ * All data is converted into PrimGroupWriterInterface objects, which are then
+ * ordered to process their data at the appropriate time.
+ * */
+
+public class BinarySerializer {
+
+    /**
+     * Interface used to write a group of primitives. One of these for each
+     * group type (Node, Way, Relation, DenseNode, Changeset)
+     */
+    protected interface PrimGroupWriterInterface {
+        /** This callback is invoked on each group that is going into the fileblock in order to give it a chance to 
+         * add to the stringtable pool of strings. */
+        public void addStringsToStringtable();
+
+        /**
+         * This callback is invoked to request that the primgroup serialize itself into the given protocol buffer object.
+         */
+        public void serialize(Osmformat.PrimitiveBlock.Builder group);
+    }
+
+    /** Set the granularity (precision of lat/lon, measured in unites of nanodegrees. */
+    public void configGranularity(int granularity) {
+        this.granularity = granularity;
+    }
+
+    /** Set whether metadata is to be omitted */
+    public void configOmit(boolean omit_metadata) {
+        this.omit_metadata = omit_metadata;
+    }
+
+    /** Configure the maximum number of entities in a batch */
+    public void configBatchLimit(int batch_limit) {
+        this.batch_limit = batch_limit;
+    }
+
+    // Paramaters affecting the output size.
+    protected final int MIN_DENSE = 10;
+    protected int batch_limit = 4000;
+
+    // Parmaters affecting the output.
+
+    protected int granularity = 100;
+    protected int date_granularity = 1000;
+    protected boolean omit_metadata = false;
+
+    /** How many primitives have been seen in this batch */
+    protected int batch_size = 0;
+    protected int total_entities = 0;
+    private StringTable stringtable = new StringTable();
+    protected List<PrimGroupWriterInterface> groups = new ArrayList<PrimGroupWriterInterface>();
+    protected BlockOutputStream output;
+
+    public BinarySerializer(BlockOutputStream output) {
+        this.output = output;
+    }
+
+    public StringTable getStringTable() {
+        return stringtable;
+    }
+
+    public void flush() throws IOException {
+        processBatch();
+        output.flush();
+    }
+
+    public void close() throws IOException {
+        flush();
+        output.close();
+    }
+
+    long debug_bytes = 0;
+
+    public void processBatch() {
+        // System.out.format("Batch of %d groups: ",groups.size());
+        if (groups.size() == 0)
+            return;
+        Osmformat.PrimitiveBlock.Builder primblock = Osmformat.PrimitiveBlock
+                .newBuilder();
+        stringtable.clear();
+        // Preprocessing: Figure out the stringtable.
+        for (PrimGroupWriterInterface i : groups)
+            i.addStringsToStringtable();
+
+        stringtable.finish();
+        // Now, start serializing.
+        for (PrimGroupWriterInterface i : groups) {
+            i.serialize(primblock);
+        }
+        primblock.setStringtable(stringtable.serialize());
+        primblock.setGranularity(this.granularity);
+        primblock.setDateGranularity(this.date_granularity);
+
+        // Only generate data with offset (0,0)
+        // 
+        Osmformat.PrimitiveBlock message = primblock.build();
+
+        // System.out.println(message);
+        debug_bytes += message.getSerializedSize();
+        if (false) // TODO: Prettyprinted output.
+            System.out.format("    =======>  %.2f / %.2f   (%dk)\n", message
+                    .getSerializedSize() / 1024.0, debug_bytes / 1024 / 1024.0,
+                    total_entities / 1000);
+        // if (message.getSerializedSize() > 1000000)
+        // System.out.println(message);
+
+        try {
+            output.write(FileBlock.newInstance("OSMData", message
+                    .toByteString(), null));
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+            throw new Error(e);
+        } finally {
+            batch_size = 0;
+            groups.clear();
+        }
+        // System.out.format("\n");
+    }
+
+    /** Convert from a degrees represented as a double into the serialized offset in nanodegrees.. */
+    public long mapRawDegrees(double degrees) {
+        return (long) ((degrees / .000000001));
+    }
+
+    /** Convert from a degrees represented as a double into the serialized offset. */
+    public int mapDegrees(double degrees) {
+        return (int) ((degrees / .0000001) / (granularity / 100));
+    }
+}

src.java/crosby/binary/StringTable.java

@@ -0,0 +1,75 @@
+package crosby.binary;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Class for mapping a set of strings to integers, giving frequently occuring
+ * strings small integers.
+ */
+public class StringTable {
+    public StringTable() {
+        clear();
+    }
+
+    private HashMap<String, Integer> counts;
+    private HashMap<String, Integer> stringmap;
+    private String set[];
+
+    public void incr(String s) {
+        if (counts.containsKey(s)) {
+            counts.put(s, new Integer(counts.get(s).intValue() + 1));
+        } else {
+            counts.put(s, new Integer(1));
+        }
+    }
+
+    public int getIndex(String s) {
+        return stringmap.get(s).intValue();
+    }
+
+    public void finish() {
+        Comparator<String> comparator = new Comparator<String>() {
+            @Override
+            public int compare(final String s1, String s2) {
+                int diff = counts.get(s2) - counts.get(s1);
+                return diff;
+            }
+        };
+
+        set = counts.keySet().toArray(new String[0]);
+        // Sort based on the frequency.
+        Arrays.sort(set, comparator);
+        // Each group of keys that serializes to the same number of bytes is
+        // sorted lexiconographically.
+        // to maximize deflate compression.
+        Arrays.sort(set, Math.min(0, set.length), Math.min(1 << 7, set.length));
+        Arrays.sort(set, Math.min(1 << 7, set.length), Math.min(1 << 14,
+                set.length));
+        Arrays.sort(set, Math.min(1 << 14, set.length), Math.min(1 << 21,
+                set.length), comparator);
+
+        stringmap = new HashMap<String, Integer>(2 * set.length);
+        for (int i = 0; i < set.length; i++) {
+            stringmap.put(set[i], new Integer(i));
+        }
+        counts = null;
+    }
+
+    public void clear() {
+        counts = new HashMap<String, Integer>(100);
+        stringmap = null;
+        set = null;
+    }
+
+    public Osmformat.StringTable.Builder serialize() {
+        Osmformat.StringTable.Builder builder = Osmformat.StringTable
+                .newBuilder();
+        for (int i = 0; i < set.length; i++)
+            builder.addS(ByteString.copyFromUtf8(set[i]));
+        return builder;
+    }
+}

src.java/crosby/binary/file/BlockInputStream.java

@@ -0,0 +1,26 @@
+package crosby.binary.file;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class BlockInputStream {
+    // TODO: Should be seekable input stream!
+    public BlockInputStream(InputStream input, BlockReaderAdapter adaptor) {
+        this.input = input;
+        this.adaptor = adaptor;
+    }
+
+    public void process() throws IOException {
+        while (input.available() > 0) {
+            FileBlock.process(input, adaptor);
+        }
+        adaptor.complete();
+    }
+
+    public void close() throws IOException {
+        input.close();
+    }
+
+    InputStream input;
+    BlockReaderAdapter adaptor;
+}

src.java/crosby/binary/file/BlockOutputStream.java

@@ -0,0 +1,60 @@
+package crosby.binary.file;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.protobuf.CodedOutputStream;
+
+enum CompressFlags {
+    NONE, DEFLATE
+}
+
+public class BlockOutputStream {
+
+    public BlockOutputStream(OutputStream output) {
+        this.outwrite = new DataOutputStream(output);
+        this.compression = CompressFlags.DEFLATE;
+    }
+
+    public void setCompress(CompressFlags flag) {
+        compression = flag;
+    }
+
+    public void setCompress(String s) {
+        if (s.equals("none"))
+            compression = CompressFlags.NONE;
+        else if (s.equals("deflate"))
+            compression = CompressFlags.DEFLATE;
+        else
+            throw new Error("Unknown compression type: " + s);
+    }
+
+    /** Write a block with the stream's default compression flag */
+    public void write(FileBlock block) throws IOException {
+        this.write(block, compression);
+    }
+
+    /** Write a specific block with a specific compression flags */
+    public void write(FileBlock block, CompressFlags compression)
+            throws IOException {
+        FileBlockPosition ref = block.writeTo(outwrite, compression);
+        writtenblocks.add(ref);
+    }
+
+    public void flush() throws IOException {
+        outwrite.flush();
+    }
+
+    public void close() throws IOException {
+        outwrite.flush();
+        outwrite.close();
+    }
+
+    OutputStream outwrite;
+    List<FileBlockPosition> writtenblocks = new ArrayList<FileBlockPosition>();
+    CompressFlags compression;
+}

src.java/crosby/binary/file/BlockReaderAdapter.java

@@ -0,0 +1,23 @@
+package crosby.binary.file;
+
+/** An adaptor that receives blocks from an input stream */
+public interface BlockReaderAdapter {
+    /**
+     * Does the reader understand this block? Does it want the data in it?
+     * 
+     * A reference contains the metadata about a block and can saved --- or
+     * stored ---- for future random access. However, during a strea read of the
+     * file, does the user want this block?
+     * 
+     * handleBlock will be called on all blocks that are not skipped, in file
+     * order.
+     * 
+     * */
+    boolean skipBlock(FileBlockPosition message);
+
+    /** Called with the data in the block. */
+    void handleBlock(FileBlock message);
+
+    /** Called when the file is fully read. */
+    void complete();
+}

src.java/crosby/binary/file/FileBlock.java

@@ -0,0 +1,110 @@
+package crosby.binary.file;
+
+import java.io.DataOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.zip.Deflater;
+
+import com.google.protobuf.ByteString;
+
+import crosby.binary.Fileformat;
+import crosby.binary.Fileformat.BlockHeader;
+
+/** A full fileblock object contains both the metadata and data of a fileblock */
+public class FileBlock extends FileBlockBase {
+    /** Contains the contents of a block for use or further processing */
+    ByteString data; // serialized Format.Blob
+
+    private FileBlock(String type, ByteString blob, ByteString indexdata) {
+        super(type, indexdata);
+        this.data = blob;
+    }
+
+    public static FileBlock newInstance(String type, ByteString blob,
+            ByteString indexdata) {
+        return new FileBlock(type, blob, indexdata);
+    }
+
+    public static FileBlock newInstance(String type, ByteString indexdata) {
+        return new FileBlock(type, null, indexdata);
+    }
+
+    protected void deflateInto(crosby.binary.Fileformat.Blob.Builder blobbuilder) {
+        int size = data.size();
+        Deflater deflater = new Deflater();
+        deflater.setInput(data.toByteArray());
+        deflater.finish();
+        byte out[] = new byte[size];
+        deflater.deflate(out);
+
+        if (!deflater.finished()) {
+            // Buffer wasn't long enough. Be noisy.
+            System.out
+                    .println("Compressed buffer too short causing extra copy");
+            out = Arrays.copyOf(out, size + size / 64 + 16);
+            deflater.deflate(out, deflater.getTotalOut(), out.length
+                    - deflater.getTotalOut());
+            assert (deflater.finished());
+        }
+        ByteString compressed = ByteString.copyFrom(out, 0, deflater
+                .getTotalOut());
+        blobbuilder.setZlibData(compressed);
+        deflater.end();
+    }
+
+    public FileBlockPosition writeTo(OutputStream outwrite, CompressFlags flags)
+            throws IOException {
+        BlockHeader.Builder builder = Fileformat.BlockHeader
+                .newBuilder();
+        if (indexdata != null)
+            builder.setIndexdata(indexdata);
+        builder.setType(type);
+
+        Fileformat.Blob.Builder blobbuilder = Fileformat.Blob.newBuilder();
+        if (flags == CompressFlags.NONE) {
+            blobbuilder.setRaw(data);
+        } else {
+            blobbuilder.setRawSize(data.size());
+            if (flags == CompressFlags.DEFLATE)
+                deflateInto(blobbuilder);
+            else
+                assert false : "TODO"; // TODO
+        }
+        Fileformat.Blob blob = blobbuilder.build();
+
+        builder.setDatasize(blob.getSerializedSize());
+        Fileformat.BlockHeader message = builder.build();
+        int size = message.getSerializedSize();
+
+        // System.out.format("Outputed header size %d bytes, header of %d bytes, and blob of %d bytes\n",
+        // size,message.getSerializedSize(),blob.getSerializedSize());
+        (new DataOutputStream(outwrite)).writeInt(size);
+        message.writeTo(outwrite);
+        long offset = -1;
+
+        if (outwrite instanceof FileOutputStream)
+            offset = ((FileOutputStream) outwrite).getChannel().position();
+
+        blob.writeTo(outwrite);
+        return FileBlockPosition.newInstance(this, offset, size);
+    }
+
+    /** Reads or skips a fileblock. */
+    static void process(InputStream input, BlockReaderAdapter callback)
+            throws IOException {
+        FileBlockHead fileblock = FileBlockHead.readHead(input);
+        if (callback.skipBlock(fileblock)) {
+            // System.out.format("Attempt to skip %d bytes\n",header.getDatasize());
+            fileblock.skipContents(input);
+        } else {
+            callback.handleBlock(fileblock.readContents(input));
+        }
+    }
+
+    public ByteString getData() {
+        return data;
+    }
+}

src.java/crosby/binary/file/FileBlockBase.java

@@ -0,0 +1,36 @@
+package crosby.binary.file;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Base class that contains the metadata about a fileblock.
+ * 
+ * Subclasses of this include additional fields, such as byte offsets that let a
+ * fileblock be read in a random-access fashion, or the data itself.
+ * 
+ * @author crosby
+ * 
+ */
+public class FileBlockBase {
+
+    protected FileBlockBase(String type, ByteString indexdata) {
+        this.type = type;
+        this.indexdata = indexdata;
+    }
+
+    /** Identifies the type of the data within a block */
+    protected final String type;
+    /**
+     * Block metadata, stored in the index block and as a prefix for every
+     * block.
+     */
+    protected final ByteString indexdata;
+
+    public String getType() {
+        return type;
+    }
+
+    public ByteString getIndexData() {
+        return indexdata;
+    }
+}

src.java/crosby/binary/file/FileBlockHead.java

@@ -0,0 +1,77 @@
+package crosby.binary.file;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+import com.google.protobuf.ByteString;
+
+import crosby.binary.Fileformat;
+
+/**
+ * Intermediate representation of the header of a fileblock when a set of
+ * fileblocks is read as in a stream. The data in the fileblock must be either
+ * skipped (where the returned value is a reference to the fileblock) or parsed.
+ * 
+ * @author crosby
+ * 
+ */
+public class FileBlockHead extends FileBlockReference {
+    protected FileBlockHead(String type, ByteString indexdata) {
+        super(type, indexdata);
+    }
+
+    /**
+     * Read the header. After reading the header, either the contents must be
+     * skipped or read
+     */
+    static FileBlockHead readHead(InputStream input) throws IOException {
+        DataInputStream datinput = new DataInputStream(input);
+        int headersize = datinput.readInt();
+        // System.out.format("Header size %d %x\n",headersize,headersize);
+        byte buf[] = new byte[headersize];
+        datinput.readFully(buf);
+        // System.out.format("Read buffer for header of %d bytes\n",buf.length);
+        Fileformat.BlockHeader header = Fileformat.BlockHeader
+                .parseFrom(buf);
+        FileBlockHead fileblock = new FileBlockHead(header.getType(), header
+                .getIndexdata());
+
+        fileblock.datasize = header.getDatasize();
+        // data_offset =
+        fileblock.input = input;
+        if (input instanceof FileInputStream)
+            fileblock.data_offset = ((FileInputStream) input).getChannel()
+                    .position();
+
+        return fileblock;
+    }
+
+    /**
+     * Assumes the stream is positioned over at the start of the data, skip over
+     * it.
+     * 
+     * @throws IOException
+     */
+    void skipContents(InputStream input) throws IOException {
+        if (input.skip(getDatasize()) != getDatasize())
+            assert false : "SHORT READ";
+    }
+
+    /**
+     * Assumes the stream is positioned over at the start of the data, read it
+     * and return the complete FileBlock
+     * 
+     * @throws IOException
+     */
+    FileBlock readContents(InputStream input) throws IOException {
+        DataInputStream datinput = new DataInputStream(input);
+        byte buf[] = new byte[getDatasize()];
+        datinput.readFully(buf);
+        return parseData(buf);
+    }
+}

src.java/crosby/binary/file/FileBlockPosition.java

@@ -0,0 +1,97 @@
+package crosby.binary.file;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+import crosby.binary.Fileformat;
+
+/**
+ * Stores the position in the stream of a fileblock so that it can be easily
+ * read in a random-access fashion.
+ * 
+ * We can turn this into a 'real' block by appropriately seeking into the file
+ * and doing a 'read'.
+ * 
+ * */
+public class FileBlockPosition extends FileBlockBase {
+    protected FileBlockPosition(String type, ByteString indexdata) {
+        super(type, indexdata);
+    }
+
+    /** Parse out and decompress the data part of a fileblock helper function. */
+    FileBlock parseData(byte buf[]) throws InvalidProtocolBufferException {
+        FileBlock out = FileBlock.newInstance(type, indexdata);
+        Fileformat.Blob blob = Fileformat.Blob.parseFrom(buf);
+        if (blob.hasRaw()) {
+            out.data = blob.getRaw();
+        } else if (blob.hasZlibData()) {
+            byte buf2[] = new byte[blob.getRawSize()];
+            Inflater decompresser = new Inflater();
+            decompresser.setInput(blob.getZlibData().toByteArray());
+            // decompresser.getRemaining();
+            try {
+                decompresser.inflate(buf2);
+            } catch (DataFormatException e) {
+                e.printStackTrace();
+                throw new Error(e);
+            }
+            assert (decompresser.finished());
+            decompresser.end();
+            out.data = ByteString.copyFrom(buf2);
+        }
+        return out;
+    }
+
+    public int getDatasize() {
+        return datasize;
+    }
+
+    /*
+     * Given any form of fileblock and an offset/length value, return a
+     * reference that can be used to dereference and read the contents.
+     */
+    static FileBlockPosition newInstance(FileBlockBase base, long offset,
+            int length) {
+        FileBlockPosition out = new FileBlockPosition(base.type, base.indexdata);
+        out.datasize = length;
+        out.data_offset = offset;
+        return out;
+    }
+
+    public FileBlock read(InputStream input) throws IOException {
+        if (input instanceof FileInputStream) {
+            ((FileInputStream) input).getChannel().position(data_offset);
+            byte buf[] = new byte[getDatasize()];
+            (new DataInputStream(input)).readFully(buf);
+            return parseData(buf);
+        } else {
+            throw new Error("Random access binary reads require seekability");
+        }
+    }
+
+    /**
+     * TODO: Convert this reference into a serialized representation that can be
+     * stored.
+     */
+    public ByteString serialize() {
+        assert false; // TODO
+        return null;
+    }
+
+    /** TODO: Parse a serialized representation of this block reference */
+    static FileBlockPosition parseFrom(ByteString b) {
+        assert false; // TODO
+        return null;
+    }
+
+    protected int datasize;
+    /** Offset into the file of the data part of the block */
+    long data_offset;
+}

src.java/crosby/binary/file/FileBlockReference.java

@@ -0,0 +1,37 @@
+package crosby.binary.file;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * A FileBlockPosition that remembers what file this is so that it can simply be
+ * dereferenced
+ */
+public class FileBlockReference extends FileBlockPosition {
+
+    /**
+     * Convenience cache for storing the input this reference is contained
+     * within so that it can be cached
+     */
+    protected InputStream input;
+
+    protected FileBlockReference(String type, ByteString indexdata) {
+        super(type, indexdata);
+    }
+
+    public FileBlock read() throws IOException {
+        return read(input);
+    }
+
+    static FileBlockPosition newInstance(FileBlockBase base, InputStream input,
+            long offset, int length) {
+        FileBlockReference out = new FileBlockReference(base.type,
+                base.indexdata);
+        out.datasize = length;
+        out.data_offset = offset;
+        out.input = input;
+        return out;
+    }
+}

src/fileformat.proto

@@ -0,0 +1,59 @@
+option java_package = "crosby.binary";
+//protoc --java_out=../.. fileformat.proto
+
+
+//
+//  STORAGE LAYER: Storing primitives.
+//
+
+message Blob {
+  optional bytes raw = 1; // No compression
+  optional int32 raw_size = 2; // When compressed, the uncompressed size
+  optional bytes zlib_data = 3;
+  optional bytes lzma_data = 4;
+  optional bytes bzip2_data = 5;
+}
+
+
+/* Future message to store an index of all of the fileblocks in a
+file. Always stored at the end of the file with its offset given in
+the FileHeader message. (If the FileDirectory message is not the last
+block, assume that we have concatenated files, and the next message is
+a file header.) */
+
+
+message FileDirectory {
+}
+
+
+/* First block in a file.  */
+message FileHeaderBlock {
+   required fixed32 version = 1;  // Future expansion - file format version number.
+   required fixed32 unused1 = 2;  // Future expansion
+   required fixed64 unused2 =3;   // Relative offset to filedirectory.
+}
+
+
+/* Author program for a file */
+message AuthorBlock {
+   required string fullname = 1; // Full name of the program (name, build number, build date, etc)
+   optional string shortname = 2; // Short name.
+   optional int64 version = 3;    // Version numbers, hopefully comparable with > 
+}
+
+
+/* A file contains an sequence of fileblock headers, each prefixed by
+their length, followed by a data block containing the actual data. 
+types staring with a "_" are reserved.
+
+The "_FileHeader" block is customarily written first.
+The "_Author" block identifies the writer and written second.
+*/
+
+message BlockHeader {
+  required string type = 1;
+  optional bytes indexdata = 2;
+  required int32 datasize = 3;
+}
+
+

src/osmformat.proto

@@ -0,0 +1,205 @@
+option java_package = "crosby.binary";
+
+/* OSM Binary file format 
+
+This is the master schema file of the OSM binary file format. This
+file is designed to support limited random-access and future
+extendability.
+
+A binary OSM file consists of a sequence of FileBlocks (please see
+fileformat.proto). The first fileblock contains a serialized instance
+of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that
+contain the primitives.
+
+Each primitiveblock is designed to be independently parsable. It
+contains a string table storing all strings in that block (keys and
+values in tags, roles in relations, usernames, etc.) as well as
+metadata containing the precision of coordinates or timestamps in that
+block.
+
+A primitiveblock contains a sequence of primitive groups, each
+containing primitives of the same type (nodes, densenodes, ways,
+relations). Coordinates are stored in signed 64-bit integers. Lat&lon
+are measured in units <granularity> nanodegrees. The default of
+granularity of 100 nanodegrees corresponds to about 1cm on the ground,
+and a full lat or lon fits into 32 bits.
+
+Converting an integer to a lattitude or longitude uses the formula:
+$OUT = IN * granularity / 10**9$. Many encoding schemes use delta
+coding when representing nodes and relations.
+
+*/
+
+//////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////
+
+/* Contains the file header. */
+
+message HeaderBlock {
+  required HeaderBBox bbox = 1;
+
+  // Author, name, and version number of the dataset in this file. (to permit
+  // patches/updates to be incrementally applied)
+  optional string datasetauthor = 16; // TODO: WANT THIS?
+  optional string datasetname = 17;  // TODO: WANT THIS?
+  optional int64 version = 18; // TODO: WANT THIS?
+
+  // Program generating this data
+  optional string writingprogram = 19;  // TODO: WANT THIS?
+
+  /* Additional tags to aid in parsing this dataset */
+  repeated string required_features = 4; // TODO: WANT THIS?
+  repeated string optional_features = 5; // TODO: WANT THIS?
+}
+
+/*
+
+Required features are features that an implementation must
+understand in order to be able to parse the OSM entities in the
+file. If a program sees a required feature that it does not
+understand, it must error out. Currently the following features are
+defined:
+
+  "DenseNodes" -- File uses dense nodes.
+
+
+Optional features are features that a file has that a program may
+exploit if it chooses to.
+
+  "Has_Metadata" -- Does the file contain author and timestamp metadata?
+  "Sort.Type_then_ID" -- Entites are sorted by type then ID.
+  "Sort.Geographic" -- Entities are in some form of geometric sort.
+
+*/
+
+/** The bounding box field in the OSM header. BBOX, as used in the OSM
+header. Units are in nanodegrees. */
+
+message HeaderBBox {
+   required sint64 left = 1;
+   required sint64 right = 2;
+   required sint64 top = 3;
+   required sint64 bottom = 4;
+}
+
+
+///////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////
+
+
+message PrimitiveBlock {
+  required StringTable stringtable = 1;
+  repeated PrimitiveGroup primitivegroup = 2;
+
+  // Granularity, units of nanodegrees, used to store coordinates in this block
+  optional int32 granularity = 17 [default=100]; 
+  // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees.
+  optional int64 lat_offset = 19 [default=0];
+  optional int64 lon_offset = 20 [default=0]; 
+
+// Granularity of dates, normally represented in units of milliseconds since the 1970 epoch.
+  optional int32 date_granularity = 18 [default=1000]; 
+
+
+  // Optional extensions, also included in index data.
+  //optional BBox bbox = 19;  // TODO: WANT THIS?
+}
+
+// Group of OSMPrimitives. All primitives in a group must be the same type.
+message PrimitiveGroup {
+  repeated Node     nodes = 1;
+  optional DenseNodes dense = 2;
+  repeated Way      ways = 3;
+  repeated Relation relations = 4;
+  repeated ChangeSet changesets = 5;
+}
+
+
+/** String table, contains the common strings in each block */
+message StringTable {
+   repeated bytes s = 1;
+}
+
+/* Optional metadata that may be included into each primitive. */
+message Info {
+   optional int32 version = 1 [default = -1];
+   optional int32 timestamp = 2;
+   optional int64 changeset = 3;
+   optional int32 uid = 4;
+   optional int32 user_sid = 5;
+}
+
+// TODO: REMOVE THIS? NOT in osmosis schema.
+message ChangeSet {
+   required int64 id = 1;
+   // Parallel arrays.
+   repeated uint32 keys = 2 [packed = true]; // String IDs.
+   repeated uint32 vals = 3 [packed = true]; // String IDs.
+
+   optional Info info = 4;
+
+   required int64 created_at = 8;
+   optional int64 closetime_delta = 9;
+   required bool open = 10;
+   optional HeaderBBox bbox = 11;
+}
+
+
+message Node {
+   required sint64 id = 1;
+   // Parallel arrays.
+   repeated uint32 keys = 2 [packed = true]; // String IDs.
+   repeated uint32 vals = 3 [packed = true]; // String IDs.
+
+   optional Info info = 4;
+
+   required sint64 lat = 8;
+   required sint64 lon = 9;
+}
+
+/* Used to densly represent a sequence of nodes that do not have any tags.
+
+We represent these nodes columnwise as four columns: ID's, lats, and
+lons, all delta coded and info's.
+
+Each array corresponds to a column of nodes. */
+message DenseNodes {
+   repeated sint64 id = 1 [packed = true]; // DELTA coded
+
+   repeated Info info = 4;
+
+   repeated sint64 lat = 8 [packed = true]; // DELTA coded
+   repeated sint64 lon = 9 [packed = true]; // DELTA coded
+}
+
+
+message Way {
+   required int64 id = 1;
+   // Parallel arrays.
+   repeated uint32 keys = 2 [packed = true];
+   repeated uint32 vals = 3 [packed = true];
+
+   optional Info info = 4;
+
+   repeated sint64 refs = 8 [packed = true];  // DELTA encoded
+}
+
+message Relation {
+  enum MemberType {
+    NODE = 0;
+    WAY = 1;
+    RELATION = 2;
+  } 
+   required int64 id = 1;
+   // Parallel arrays.
+   repeated uint32 keys = 2 [packed = true];
+   repeated uint32 vals = 3 [packed = true];
+
+   optional Info info = 4;
+
+   // Parallel arrays
+   repeated int32 roles_sid = 8 [packed = true];
+   repeated sint64 memids = 9 [packed = true]; // DELTA encoded
+   repeated MemberType types = 10 [packed = true];
+}
+