From 8f0c4a8bc2fe7b6669e7550854deb497cc253c28 Mon Sep 17 00:00:00 2001 From: Scott Crosby Date: Sun, 8 Aug 2010 23:25:23 -0500 Subject: [PATCH] Protobufs and common Java code for binary format. --- build.sh | 2 + src.java/crosby/binary/BinaryParser.java | 109 ++++++++++ src.java/crosby/binary/BinarySerializer.java | 145 +++++++++++++ src.java/crosby/binary/StringTable.java | 75 +++++++ .../crosby/binary/file/BlockInputStream.java | 26 +++ .../crosby/binary/file/BlockOutputStream.java | 60 +++++ .../binary/file/BlockReaderAdapter.java | 23 ++ src.java/crosby/binary/file/FileBlock.java | 110 ++++++++++ .../crosby/binary/file/FileBlockBase.java | 36 +++ .../crosby/binary/file/FileBlockHead.java | 77 +++++++ .../crosby/binary/file/FileBlockPosition.java | 97 +++++++++ .../binary/file/FileBlockReference.java | 37 ++++ src/fileformat.proto | 59 +++++ src/osmformat.proto | 205 ++++++++++++++++++ 14 files changed, 1061 insertions(+) create mode 100644 build.sh create mode 100644 src.java/crosby/binary/BinaryParser.java create mode 100644 src.java/crosby/binary/BinarySerializer.java create mode 100644 src.java/crosby/binary/StringTable.java create mode 100644 src.java/crosby/binary/file/BlockInputStream.java create mode 100644 src.java/crosby/binary/file/BlockOutputStream.java create mode 100644 src.java/crosby/binary/file/BlockReaderAdapter.java create mode 100644 src.java/crosby/binary/file/FileBlock.java create mode 100644 src.java/crosby/binary/file/FileBlockBase.java create mode 100644 src.java/crosby/binary/file/FileBlockHead.java create mode 100644 src.java/crosby/binary/file/FileBlockPosition.java create mode 100644 src.java/crosby/binary/file/FileBlockReference.java create mode 100644 src/fileformat.proto create mode 100644 src/osmformat.proto diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..06bfa27 --- /dev/null +++ b/build.sh @@ -0,0 +1,2 @@ +protoc --java_out=generated.java src/osmformat.proto +protoc --java_out=generated.java src/fileformat.proto diff --git a/src.java/crosby/binary/BinaryParser.java b/src.java/crosby/binary/BinaryParser.java new file mode 100644 index 0000000..e849fd2 --- /dev/null +++ b/src.java/crosby/binary/BinaryParser.java @@ -0,0 +1,109 @@ +package crosby.binary; + + +import java.util.Date; +import java.util.List; + +import com.google.protobuf.InvalidProtocolBufferException; + +import crosby.binary.Osmformat; +import crosby.binary.file.BlockReaderAdapter; +import crosby.binary.file.FileBlock; +import crosby.binary.file.FileBlockPosition; + +public abstract class BinaryParser implements BlockReaderAdapter { + private int granularity; + private long lat_offset; + private long lon_offset; + private int date_granularity; + private String strings[]; + + protected Date getDate(Osmformat.Info info) { + if (info.hasTimestamp()) { + return new Date(date_granularity * (long) info.getTimestamp()); + } else + return NODATE; + } + public static final Date NODATE = new Date(); + + protected String getStringById(int id) { + return strings[id]; + } + + @Override + public void handleBlock(FileBlock message) { + // TODO Auto-generated method stub + try { + if (message.getType().equals("OSMHeader")) { + Osmformat.HeaderBlock headerblock = Osmformat.HeaderBlock + .parseFrom(message.getData()); + parse(headerblock); + } else if (message.getType().equals("OSMData")) { + Osmformat.PrimitiveBlock primblock = Osmformat.PrimitiveBlock + .parseFrom(message.getData()); + parse(primblock); + } + } catch (InvalidProtocolBufferException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new Error("ParseError"); // TODO + } + + } + + + @Override + public boolean skipBlock(FileBlockPosition block) { + // System.out.println("Seeing block of type: "+block.getType()); + if (block.getType().equals("OSMData")) + return false; + if (block.getType().equals("OSMHeader")) + return false; + System.out.println("Skipped block of type: " + block.getType()); + return true; + } + + + + public double parseLat(long degree) { + // Support non-zero offsets. (We don't currently generate them) + return (granularity * degree + lat_offset) * .000000001; + } + + public double parseLon(long degree) { + // Support non-zero offsets. (We don't currently generate them) + return (granularity * degree + lon_offset) * .000000001; + } + + + public void parse(Osmformat.PrimitiveBlock block) { + Osmformat.StringTable stablemessage = block.getStringtable(); + strings = new String[stablemessage.getSCount()]; + + for (int i = 0; i < strings.length; i++) { + strings[i] = stablemessage.getS(i).toStringUtf8(); + } + + granularity = block.getGranularity(); + lat_offset = block.getLatOffset(); + lon_offset = block.getLonOffset(); + date_granularity = block.getDateGranularity(); + + for (Osmformat.PrimitiveGroup groupmessage : block + .getPrimitivegroupList()) { + // Exactly one of these should trigger on each loop. + parseNodes(groupmessage.getNodesList()); + parseWays(groupmessage.getWaysList()); + parseRelations(groupmessage.getRelationsList()); + if (groupmessage.hasDense()) + parseDense(groupmessage.getDense()); + } + } + + protected abstract void parseRelations(List rels); + protected abstract void parseDense(Osmformat.DenseNodes nodes); + protected abstract void parseNodes(List nodes); + protected abstract void parseWays(List ways); + protected abstract void parse(Osmformat.HeaderBlock header); + +} \ No newline at end of file diff --git a/src.java/crosby/binary/BinarySerializer.java b/src.java/crosby/binary/BinarySerializer.java new file mode 100644 index 0000000..a690c17 --- /dev/null +++ b/src.java/crosby/binary/BinarySerializer.java @@ -0,0 +1,145 @@ +package crosby.binary; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import crosby.binary.file.BlockOutputStream; +import crosby.binary.file.FileBlock; + +/** + * Generic serializer common code + * + * Serialize a set of blobs and process them. Subclasses implement handlers for + * different API's (osmosis, mkgmap, splitter, etc.) + * + * All data is converted into PrimGroupWriterInterface objects, which are then + * ordered to process their data at the appropriate time. + * */ + +public class BinarySerializer { + + /** + * Interface used to write a group of primitives. One of these for each + * group type (Node, Way, Relation, DenseNode, Changeset) + */ + protected interface PrimGroupWriterInterface { + /** This callback is invoked on each group that is going into the fileblock in order to give it a chance to + * add to the stringtable pool of strings. */ + public void addStringsToStringtable(); + + /** + * This callback is invoked to request that the primgroup serialize itself into the given protocol buffer object. + */ + public void serialize(Osmformat.PrimitiveBlock.Builder group); + } + + /** Set the granularity (precision of lat/lon, measured in unites of nanodegrees. */ + public void configGranularity(int granularity) { + this.granularity = granularity; + } + + /** Set whether metadata is to be omitted */ + public void configOmit(boolean omit_metadata) { + this.omit_metadata = omit_metadata; + } + + /** Configure the maximum number of entities in a batch */ + public void configBatchLimit(int batch_limit) { + this.batch_limit = batch_limit; + } + + // Paramaters affecting the output size. + protected final int MIN_DENSE = 10; + protected int batch_limit = 4000; + + // Parmaters affecting the output. + + protected int granularity = 100; + protected int date_granularity = 1000; + protected boolean omit_metadata = false; + + /** How many primitives have been seen in this batch */ + protected int batch_size = 0; + protected int total_entities = 0; + private StringTable stringtable = new StringTable(); + protected List groups = new ArrayList(); + protected BlockOutputStream output; + + public BinarySerializer(BlockOutputStream output) { + this.output = output; + } + + public StringTable getStringTable() { + return stringtable; + } + + public void flush() throws IOException { + processBatch(); + output.flush(); + } + + public void close() throws IOException { + flush(); + output.close(); + } + + long debug_bytes = 0; + + public void processBatch() { + // System.out.format("Batch of %d groups: ",groups.size()); + if (groups.size() == 0) + return; + Osmformat.PrimitiveBlock.Builder primblock = Osmformat.PrimitiveBlock + .newBuilder(); + stringtable.clear(); + // Preprocessing: Figure out the stringtable. + for (PrimGroupWriterInterface i : groups) + i.addStringsToStringtable(); + + stringtable.finish(); + // Now, start serializing. + for (PrimGroupWriterInterface i : groups) { + i.serialize(primblock); + } + primblock.setStringtable(stringtable.serialize()); + primblock.setGranularity(this.granularity); + primblock.setDateGranularity(this.date_granularity); + + // Only generate data with offset (0,0) + // + Osmformat.PrimitiveBlock message = primblock.build(); + + // System.out.println(message); + debug_bytes += message.getSerializedSize(); + if (false) // TODO: Prettyprinted output. + System.out.format(" =======> %.2f / %.2f (%dk)\n", message + .getSerializedSize() / 1024.0, debug_bytes / 1024 / 1024.0, + total_entities / 1000); + // if (message.getSerializedSize() > 1000000) + // System.out.println(message); + + try { + output.write(FileBlock.newInstance("OSMData", message + .toByteString(), null)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + throw new Error(e); + } finally { + batch_size = 0; + groups.clear(); + } + // System.out.format("\n"); + } + + /** Convert from a degrees represented as a double into the serialized offset in nanodegrees.. */ + public long mapRawDegrees(double degrees) { + return (long) ((degrees / .000000001)); + } + + /** Convert from a degrees represented as a double into the serialized offset. */ + public int mapDegrees(double degrees) { + return (int) ((degrees / .0000001) / (granularity / 100)); + } +} diff --git a/src.java/crosby/binary/StringTable.java b/src.java/crosby/binary/StringTable.java new file mode 100644 index 0000000..04bc9d6 --- /dev/null +++ b/src.java/crosby/binary/StringTable.java @@ -0,0 +1,75 @@ +package crosby.binary; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; + +import com.google.protobuf.ByteString; + +/** + * Class for mapping a set of strings to integers, giving frequently occuring + * strings small integers. + */ +public class StringTable { + public StringTable() { + clear(); + } + + private HashMap counts; + private HashMap stringmap; + private String set[]; + + public void incr(String s) { + if (counts.containsKey(s)) { + counts.put(s, new Integer(counts.get(s).intValue() + 1)); + } else { + counts.put(s, new Integer(1)); + } + } + + public int getIndex(String s) { + return stringmap.get(s).intValue(); + } + + public void finish() { + Comparator comparator = new Comparator() { + @Override + public int compare(final String s1, String s2) { + int diff = counts.get(s2) - counts.get(s1); + return diff; + } + }; + + set = counts.keySet().toArray(new String[0]); + // Sort based on the frequency. + Arrays.sort(set, comparator); + // Each group of keys that serializes to the same number of bytes is + // sorted lexiconographically. + // to maximize deflate compression. + Arrays.sort(set, Math.min(0, set.length), Math.min(1 << 7, set.length)); + Arrays.sort(set, Math.min(1 << 7, set.length), Math.min(1 << 14, + set.length)); + Arrays.sort(set, Math.min(1 << 14, set.length), Math.min(1 << 21, + set.length), comparator); + + stringmap = new HashMap(2 * set.length); + for (int i = 0; i < set.length; i++) { + stringmap.put(set[i], new Integer(i)); + } + counts = null; + } + + public void clear() { + counts = new HashMap(100); + stringmap = null; + set = null; + } + + public Osmformat.StringTable.Builder serialize() { + Osmformat.StringTable.Builder builder = Osmformat.StringTable + .newBuilder(); + for (int i = 0; i < set.length; i++) + builder.addS(ByteString.copyFromUtf8(set[i])); + return builder; + } +} diff --git a/src.java/crosby/binary/file/BlockInputStream.java b/src.java/crosby/binary/file/BlockInputStream.java new file mode 100644 index 0000000..238c201 --- /dev/null +++ b/src.java/crosby/binary/file/BlockInputStream.java @@ -0,0 +1,26 @@ +package crosby.binary.file; + +import java.io.IOException; +import java.io.InputStream; + +public class BlockInputStream { + // TODO: Should be seekable input stream! + public BlockInputStream(InputStream input, BlockReaderAdapter adaptor) { + this.input = input; + this.adaptor = adaptor; + } + + public void process() throws IOException { + while (input.available() > 0) { + FileBlock.process(input, adaptor); + } + adaptor.complete(); + } + + public void close() throws IOException { + input.close(); + } + + InputStream input; + BlockReaderAdapter adaptor; +} diff --git a/src.java/crosby/binary/file/BlockOutputStream.java b/src.java/crosby/binary/file/BlockOutputStream.java new file mode 100644 index 0000000..10623b0 --- /dev/null +++ b/src.java/crosby/binary/file/BlockOutputStream.java @@ -0,0 +1,60 @@ +package crosby.binary.file; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.List; + +import com.google.protobuf.CodedOutputStream; + +enum CompressFlags { + NONE, DEFLATE +} + +public class BlockOutputStream { + + public BlockOutputStream(OutputStream output) { + this.outwrite = new DataOutputStream(output); + this.compression = CompressFlags.DEFLATE; + } + + public void setCompress(CompressFlags flag) { + compression = flag; + } + + public void setCompress(String s) { + if (s.equals("none")) + compression = CompressFlags.NONE; + else if (s.equals("deflate")) + compression = CompressFlags.DEFLATE; + else + throw new Error("Unknown compression type: " + s); + } + + /** Write a block with the stream's default compression flag */ + public void write(FileBlock block) throws IOException { + this.write(block, compression); + } + + /** Write a specific block with a specific compression flags */ + public void write(FileBlock block, CompressFlags compression) + throws IOException { + FileBlockPosition ref = block.writeTo(outwrite, compression); + writtenblocks.add(ref); + } + + public void flush() throws IOException { + outwrite.flush(); + } + + public void close() throws IOException { + outwrite.flush(); + outwrite.close(); + } + + OutputStream outwrite; + List writtenblocks = new ArrayList(); + CompressFlags compression; +} diff --git a/src.java/crosby/binary/file/BlockReaderAdapter.java b/src.java/crosby/binary/file/BlockReaderAdapter.java new file mode 100644 index 0000000..4ef4759 --- /dev/null +++ b/src.java/crosby/binary/file/BlockReaderAdapter.java @@ -0,0 +1,23 @@ +package crosby.binary.file; + +/** An adaptor that receives blocks from an input stream */ +public interface BlockReaderAdapter { + /** + * Does the reader understand this block? Does it want the data in it? + * + * A reference contains the metadata about a block and can saved --- or + * stored ---- for future random access. However, during a strea read of the + * file, does the user want this block? + * + * handleBlock will be called on all blocks that are not skipped, in file + * order. + * + * */ + boolean skipBlock(FileBlockPosition message); + + /** Called with the data in the block. */ + void handleBlock(FileBlock message); + + /** Called when the file is fully read. */ + void complete(); +} diff --git a/src.java/crosby/binary/file/FileBlock.java b/src.java/crosby/binary/file/FileBlock.java new file mode 100644 index 0000000..03378c2 --- /dev/null +++ b/src.java/crosby/binary/file/FileBlock.java @@ -0,0 +1,110 @@ +package crosby.binary.file; + +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.zip.Deflater; + +import com.google.protobuf.ByteString; + +import crosby.binary.Fileformat; +import crosby.binary.Fileformat.BlockHeader; + +/** A full fileblock object contains both the metadata and data of a fileblock */ +public class FileBlock extends FileBlockBase { + /** Contains the contents of a block for use or further processing */ + ByteString data; // serialized Format.Blob + + private FileBlock(String type, ByteString blob, ByteString indexdata) { + super(type, indexdata); + this.data = blob; + } + + public static FileBlock newInstance(String type, ByteString blob, + ByteString indexdata) { + return new FileBlock(type, blob, indexdata); + } + + public static FileBlock newInstance(String type, ByteString indexdata) { + return new FileBlock(type, null, indexdata); + } + + protected void deflateInto(crosby.binary.Fileformat.Blob.Builder blobbuilder) { + int size = data.size(); + Deflater deflater = new Deflater(); + deflater.setInput(data.toByteArray()); + deflater.finish(); + byte out[] = new byte[size]; + deflater.deflate(out); + + if (!deflater.finished()) { + // Buffer wasn't long enough. Be noisy. + System.out + .println("Compressed buffer too short causing extra copy"); + out = Arrays.copyOf(out, size + size / 64 + 16); + deflater.deflate(out, deflater.getTotalOut(), out.length + - deflater.getTotalOut()); + assert (deflater.finished()); + } + ByteString compressed = ByteString.copyFrom(out, 0, deflater + .getTotalOut()); + blobbuilder.setZlibData(compressed); + deflater.end(); + } + + public FileBlockPosition writeTo(OutputStream outwrite, CompressFlags flags) + throws IOException { + BlockHeader.Builder builder = Fileformat.BlockHeader + .newBuilder(); + if (indexdata != null) + builder.setIndexdata(indexdata); + builder.setType(type); + + Fileformat.Blob.Builder blobbuilder = Fileformat.Blob.newBuilder(); + if (flags == CompressFlags.NONE) { + blobbuilder.setRaw(data); + } else { + blobbuilder.setRawSize(data.size()); + if (flags == CompressFlags.DEFLATE) + deflateInto(blobbuilder); + else + assert false : "TODO"; // TODO + } + Fileformat.Blob blob = blobbuilder.build(); + + builder.setDatasize(blob.getSerializedSize()); + Fileformat.BlockHeader message = builder.build(); + int size = message.getSerializedSize(); + + // System.out.format("Outputed header size %d bytes, header of %d bytes, and blob of %d bytes\n", + // size,message.getSerializedSize(),blob.getSerializedSize()); + (new DataOutputStream(outwrite)).writeInt(size); + message.writeTo(outwrite); + long offset = -1; + + if (outwrite instanceof FileOutputStream) + offset = ((FileOutputStream) outwrite).getChannel().position(); + + blob.writeTo(outwrite); + return FileBlockPosition.newInstance(this, offset, size); + } + + /** Reads or skips a fileblock. */ + static void process(InputStream input, BlockReaderAdapter callback) + throws IOException { + FileBlockHead fileblock = FileBlockHead.readHead(input); + if (callback.skipBlock(fileblock)) { + // System.out.format("Attempt to skip %d bytes\n",header.getDatasize()); + fileblock.skipContents(input); + } else { + callback.handleBlock(fileblock.readContents(input)); + } + } + + public ByteString getData() { + return data; + } +} diff --git a/src.java/crosby/binary/file/FileBlockBase.java b/src.java/crosby/binary/file/FileBlockBase.java new file mode 100644 index 0000000..33644cc --- /dev/null +++ b/src.java/crosby/binary/file/FileBlockBase.java @@ -0,0 +1,36 @@ +package crosby.binary.file; + +import com.google.protobuf.ByteString; + +/** + * Base class that contains the metadata about a fileblock. + * + * Subclasses of this include additional fields, such as byte offsets that let a + * fileblock be read in a random-access fashion, or the data itself. + * + * @author crosby + * + */ +public class FileBlockBase { + + protected FileBlockBase(String type, ByteString indexdata) { + this.type = type; + this.indexdata = indexdata; + } + + /** Identifies the type of the data within a block */ + protected final String type; + /** + * Block metadata, stored in the index block and as a prefix for every + * block. + */ + protected final ByteString indexdata; + + public String getType() { + return type; + } + + public ByteString getIndexData() { + return indexdata; + } +} diff --git a/src.java/crosby/binary/file/FileBlockHead.java b/src.java/crosby/binary/file/FileBlockHead.java new file mode 100644 index 0000000..2798f07 --- /dev/null +++ b/src.java/crosby/binary/file/FileBlockHead.java @@ -0,0 +1,77 @@ +package crosby.binary.file; + +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import com.google.protobuf.ByteString; + +import crosby.binary.Fileformat; + +/** + * Intermediate representation of the header of a fileblock when a set of + * fileblocks is read as in a stream. The data in the fileblock must be either + * skipped (where the returned value is a reference to the fileblock) or parsed. + * + * @author crosby + * + */ +public class FileBlockHead extends FileBlockReference { + protected FileBlockHead(String type, ByteString indexdata) { + super(type, indexdata); + } + + /** + * Read the header. After reading the header, either the contents must be + * skipped or read + */ + static FileBlockHead readHead(InputStream input) throws IOException { + DataInputStream datinput = new DataInputStream(input); + int headersize = datinput.readInt(); + // System.out.format("Header size %d %x\n",headersize,headersize); + byte buf[] = new byte[headersize]; + datinput.readFully(buf); + // System.out.format("Read buffer for header of %d bytes\n",buf.length); + Fileformat.BlockHeader header = Fileformat.BlockHeader + .parseFrom(buf); + FileBlockHead fileblock = new FileBlockHead(header.getType(), header + .getIndexdata()); + + fileblock.datasize = header.getDatasize(); + // data_offset = + fileblock.input = input; + if (input instanceof FileInputStream) + fileblock.data_offset = ((FileInputStream) input).getChannel() + .position(); + + return fileblock; + } + + /** + * Assumes the stream is positioned over at the start of the data, skip over + * it. + * + * @throws IOException + */ + void skipContents(InputStream input) throws IOException { + if (input.skip(getDatasize()) != getDatasize()) + assert false : "SHORT READ"; + } + + /** + * Assumes the stream is positioned over at the start of the data, read it + * and return the complete FileBlock + * + * @throws IOException + */ + FileBlock readContents(InputStream input) throws IOException { + DataInputStream datinput = new DataInputStream(input); + byte buf[] = new byte[getDatasize()]; + datinput.readFully(buf); + return parseData(buf); + } +} diff --git a/src.java/crosby/binary/file/FileBlockPosition.java b/src.java/crosby/binary/file/FileBlockPosition.java new file mode 100644 index 0000000..51a11a6 --- /dev/null +++ b/src.java/crosby/binary/file/FileBlockPosition.java @@ -0,0 +1,97 @@ +package crosby.binary.file; + +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +import crosby.binary.Fileformat; + +/** + * Stores the position in the stream of a fileblock so that it can be easily + * read in a random-access fashion. + * + * We can turn this into a 'real' block by appropriately seeking into the file + * and doing a 'read'. + * + * */ +public class FileBlockPosition extends FileBlockBase { + protected FileBlockPosition(String type, ByteString indexdata) { + super(type, indexdata); + } + + /** Parse out and decompress the data part of a fileblock helper function. */ + FileBlock parseData(byte buf[]) throws InvalidProtocolBufferException { + FileBlock out = FileBlock.newInstance(type, indexdata); + Fileformat.Blob blob = Fileformat.Blob.parseFrom(buf); + if (blob.hasRaw()) { + out.data = blob.getRaw(); + } else if (blob.hasZlibData()) { + byte buf2[] = new byte[blob.getRawSize()]; + Inflater decompresser = new Inflater(); + decompresser.setInput(blob.getZlibData().toByteArray()); + // decompresser.getRemaining(); + try { + decompresser.inflate(buf2); + } catch (DataFormatException e) { + e.printStackTrace(); + throw new Error(e); + } + assert (decompresser.finished()); + decompresser.end(); + out.data = ByteString.copyFrom(buf2); + } + return out; + } + + public int getDatasize() { + return datasize; + } + + /* + * Given any form of fileblock and an offset/length value, return a + * reference that can be used to dereference and read the contents. + */ + static FileBlockPosition newInstance(FileBlockBase base, long offset, + int length) { + FileBlockPosition out = new FileBlockPosition(base.type, base.indexdata); + out.datasize = length; + out.data_offset = offset; + return out; + } + + public FileBlock read(InputStream input) throws IOException { + if (input instanceof FileInputStream) { + ((FileInputStream) input).getChannel().position(data_offset); + byte buf[] = new byte[getDatasize()]; + (new DataInputStream(input)).readFully(buf); + return parseData(buf); + } else { + throw new Error("Random access binary reads require seekability"); + } + } + + /** + * TODO: Convert this reference into a serialized representation that can be + * stored. + */ + public ByteString serialize() { + assert false; // TODO + return null; + } + + /** TODO: Parse a serialized representation of this block reference */ + static FileBlockPosition parseFrom(ByteString b) { + assert false; // TODO + return null; + } + + protected int datasize; + /** Offset into the file of the data part of the block */ + long data_offset; +} diff --git a/src.java/crosby/binary/file/FileBlockReference.java b/src.java/crosby/binary/file/FileBlockReference.java new file mode 100644 index 0000000..508d0c4 --- /dev/null +++ b/src.java/crosby/binary/file/FileBlockReference.java @@ -0,0 +1,37 @@ +package crosby.binary.file; + +import java.io.IOException; +import java.io.InputStream; + +import com.google.protobuf.ByteString; + +/** + * A FileBlockPosition that remembers what file this is so that it can simply be + * dereferenced + */ +public class FileBlockReference extends FileBlockPosition { + + /** + * Convenience cache for storing the input this reference is contained + * within so that it can be cached + */ + protected InputStream input; + + protected FileBlockReference(String type, ByteString indexdata) { + super(type, indexdata); + } + + public FileBlock read() throws IOException { + return read(input); + } + + static FileBlockPosition newInstance(FileBlockBase base, InputStream input, + long offset, int length) { + FileBlockReference out = new FileBlockReference(base.type, + base.indexdata); + out.datasize = length; + out.data_offset = offset; + out.input = input; + return out; + } +} diff --git a/src/fileformat.proto b/src/fileformat.proto new file mode 100644 index 0000000..d6e3a2e --- /dev/null +++ b/src/fileformat.proto @@ -0,0 +1,59 @@ +option java_package = "crosby.binary"; +//protoc --java_out=../.. fileformat.proto + + +// +// STORAGE LAYER: Storing primitives. +// + +message Blob { + optional bytes raw = 1; // No compression + optional int32 raw_size = 2; // When compressed, the uncompressed size + optional bytes zlib_data = 3; + optional bytes lzma_data = 4; + optional bytes bzip2_data = 5; +} + + +/* Future message to store an index of all of the fileblocks in a +file. Always stored at the end of the file with its offset given in +the FileHeader message. (If the FileDirectory message is not the last +block, assume that we have concatenated files, and the next message is +a file header.) */ + + +message FileDirectory { +} + + +/* First block in a file. */ +message FileHeaderBlock { + required fixed32 version = 1; // Future expansion - file format version number. + required fixed32 unused1 = 2; // Future expansion + required fixed64 unused2 =3; // Relative offset to filedirectory. +} + + +/* Author program for a file */ +message AuthorBlock { + required string fullname = 1; // Full name of the program (name, build number, build date, etc) + optional string shortname = 2; // Short name. + optional int64 version = 3; // Version numbers, hopefully comparable with > +} + + +/* A file contains an sequence of fileblock headers, each prefixed by +their length, followed by a data block containing the actual data. +types staring with a "_" are reserved. + +The "_FileHeader" block is customarily written first. +The "_Author" block identifies the writer and written second. +*/ + +message BlockHeader { + required string type = 1; + optional bytes indexdata = 2; + required int32 datasize = 3; +} + + diff --git a/src/osmformat.proto b/src/osmformat.proto new file mode 100644 index 0000000..bf6e9fc --- /dev/null +++ b/src/osmformat.proto @@ -0,0 +1,205 @@ +option java_package = "crosby.binary"; + +/* OSM Binary file format + +This is the master schema file of the OSM binary file format. This +file is designed to support limited random-access and future +extendability. + +A binary OSM file consists of a sequence of FileBlocks (please see +fileformat.proto). The first fileblock contains a serialized instance +of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that +contain the primitives. + +Each primitiveblock is designed to be independently parsable. It +contains a string table storing all strings in that block (keys and +values in tags, roles in relations, usernames, etc.) as well as +metadata containing the precision of coordinates or timestamps in that +block. + +A primitiveblock contains a sequence of primitive groups, each +containing primitives of the same type (nodes, densenodes, ways, +relations). Coordinates are stored in signed 64-bit integers. Lat&lon +are measured in units nanodegrees. The default of +granularity of 100 nanodegrees corresponds to about 1cm on the ground, +and a full lat or lon fits into 32 bits. + +Converting an integer to a lattitude or longitude uses the formula: +$OUT = IN * granularity / 10**9$. Many encoding schemes use delta +coding when representing nodes and relations. + +*/ + +////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////// + +/* Contains the file header. */ + +message HeaderBlock { + required HeaderBBox bbox = 1; + + // Author, name, and version number of the dataset in this file. (to permit + // patches/updates to be incrementally applied) + optional string datasetauthor = 16; // TODO: WANT THIS? + optional string datasetname = 17; // TODO: WANT THIS? + optional int64 version = 18; // TODO: WANT THIS? + + // Program generating this data + optional string writingprogram = 19; // TODO: WANT THIS? + + /* Additional tags to aid in parsing this dataset */ + repeated string required_features = 4; // TODO: WANT THIS? + repeated string optional_features = 5; // TODO: WANT THIS? +} + +/* + +Required features are features that an implementation must +understand in order to be able to parse the OSM entities in the +file. If a program sees a required feature that it does not +understand, it must error out. Currently the following features are +defined: + + "DenseNodes" -- File uses dense nodes. + + +Optional features are features that a file has that a program may +exploit if it chooses to. + + "Has_Metadata" -- Does the file contain author and timestamp metadata? + "Sort.Type_then_ID" -- Entites are sorted by type then ID. + "Sort.Geographic" -- Entities are in some form of geometric sort. + +*/ + +/** The bounding box field in the OSM header. BBOX, as used in the OSM +header. Units are in nanodegrees. */ + +message HeaderBBox { + required sint64 left = 1; + required sint64 right = 2; + required sint64 top = 3; + required sint64 bottom = 4; +} + + +/////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////// + + +message PrimitiveBlock { + required StringTable stringtable = 1; + repeated PrimitiveGroup primitivegroup = 2; + + // Granularity, units of nanodegrees, used to store coordinates in this block + optional int32 granularity = 17 [default=100]; + // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees. + optional int64 lat_offset = 19 [default=0]; + optional int64 lon_offset = 20 [default=0]; + +// Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. + optional int32 date_granularity = 18 [default=1000]; + + + // Optional extensions, also included in index data. + //optional BBox bbox = 19; // TODO: WANT THIS? +} + +// Group of OSMPrimitives. All primitives in a group must be the same type. +message PrimitiveGroup { + repeated Node nodes = 1; + optional DenseNodes dense = 2; + repeated Way ways = 3; + repeated Relation relations = 4; + repeated ChangeSet changesets = 5; +} + + +/** String table, contains the common strings in each block */ +message StringTable { + repeated bytes s = 1; +} + +/* Optional metadata that may be included into each primitive. */ +message Info { + optional int32 version = 1 [default = -1]; + optional int32 timestamp = 2; + optional int64 changeset = 3; + optional int32 uid = 4; + optional int32 user_sid = 5; +} + +// TODO: REMOVE THIS? NOT in osmosis schema. +message ChangeSet { + required int64 id = 1; + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; // String IDs. + repeated uint32 vals = 3 [packed = true]; // String IDs. + + optional Info info = 4; + + required int64 created_at = 8; + optional int64 closetime_delta = 9; + required bool open = 10; + optional HeaderBBox bbox = 11; +} + + +message Node { + required sint64 id = 1; + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; // String IDs. + repeated uint32 vals = 3 [packed = true]; // String IDs. + + optional Info info = 4; + + required sint64 lat = 8; + required sint64 lon = 9; +} + +/* Used to densly represent a sequence of nodes that do not have any tags. + +We represent these nodes columnwise as four columns: ID's, lats, and +lons, all delta coded and info's. + +Each array corresponds to a column of nodes. */ +message DenseNodes { + repeated sint64 id = 1 [packed = true]; // DELTA coded + + repeated Info info = 4; + + repeated sint64 lat = 8 [packed = true]; // DELTA coded + repeated sint64 lon = 9 [packed = true]; // DELTA coded +} + + +message Way { + required int64 id = 1; + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; + repeated uint32 vals = 3 [packed = true]; + + optional Info info = 4; + + repeated sint64 refs = 8 [packed = true]; // DELTA encoded +} + +message Relation { + enum MemberType { + NODE = 0; + WAY = 1; + RELATION = 2; + } + required int64 id = 1; + // Parallel arrays. + repeated uint32 keys = 2 [packed = true]; + repeated uint32 vals = 3 [packed = true]; + + optional Info info = 4; + + // Parallel arrays + repeated int32 roles_sid = 8 [packed = true]; + repeated sint64 memids = 9 [packed = true]; // DELTA encoded + repeated MemberType types = 10 [packed = true]; +} +