diff --git a/src.java/crosby/binary/BinaryParser.java b/src.java/crosby/binary/BinaryParser.java index e849fd2..d0dcde6 100644 --- a/src.java/crosby/binary/BinaryParser.java +++ b/src.java/crosby/binary/BinaryParser.java @@ -26,6 +26,12 @@ public abstract class BinaryParser implements BlockReaderAdapter { } public static final Date NODATE = new Date(); + /** Get a string based on the index used. + * + * Index 0 is reserved to use as a delimiter, therefore, index 1 corresponds to the first string in the table + * @param id + * @return + */ protected String getStringById(int id) { return strings[id]; } diff --git a/src.java/crosby/binary/StringTable.java b/src.java/crosby/binary/StringTable.java index 04bc9d6..7c34dcf 100644 --- a/src.java/crosby/binary/StringTable.java +++ b/src.java/crosby/binary/StringTable.java @@ -27,6 +27,12 @@ public class StringTable { } } + /** After the stringtable has been built, return the offset of a string in it. + * + * Note, value '0' is reserved for use as a delimiter and will not be returned. + * @param s + * @return + */ public int getIndex(String s) { return stringmap.get(s).intValue(); } @@ -41,20 +47,21 @@ public class StringTable { }; set = counts.keySet().toArray(new String[0]); - // Sort based on the frequency. - Arrays.sort(set, comparator); - // Each group of keys that serializes to the same number of bytes is - // sorted lexiconographically. - // to maximize deflate compression. - Arrays.sort(set, Math.min(0, set.length), Math.min(1 << 7, set.length)); - Arrays.sort(set, Math.min(1 << 7, set.length), Math.min(1 << 14, - set.length)); - Arrays.sort(set, Math.min(1 << 14, set.length), Math.min(1 << 21, - set.length), comparator); - + if (set.length > 0) { + // Sort based on the frequency. + Arrays.sort(set, comparator); + // Each group of keys that serializes to the same number of bytes is + // sorted lexiconographically. + // to maximize deflate compression. + Arrays.sort(set, Math.min(0, set.length-1), Math.min(1 << 7, set.length-1)); + Arrays.sort(set, Math.min(1 << 7, set.length-1), Math.min(1 << 14, + set.length-1)); + Arrays.sort(set, Math.min(1 << 14, set.length-1), Math.min(1 << 21, + set.length-1), comparator); + } stringmap = new HashMap(2 * set.length); for (int i = 0; i < set.length; i++) { - stringmap.put(set[i], new Integer(i)); + stringmap.put(set[i], new Integer(i+1)); // Index 0 is reserved for use as a delimiter. } counts = null; } @@ -68,6 +75,7 @@ public class StringTable { public Osmformat.StringTable.Builder serialize() { Osmformat.StringTable.Builder builder = Osmformat.StringTable .newBuilder(); + builder.addS(ByteString.copyFromUtf8("")); // Add a unused string at offset 0 which is used as a delimiter. for (int i = 0; i < set.length; i++) builder.addS(ByteString.copyFromUtf8(set[i])); return builder; diff --git a/src/osmformat.proto b/src/osmformat.proto index bf6e9fc..760ea62 100644 --- a/src/osmformat.proto +++ b/src/osmformat.proto @@ -115,7 +115,12 @@ message PrimitiveGroup { } -/** String table, contains the common strings in each block */ +/** String table, contains the common strings in each block. + + Note that we reserve index '0' as a delimiter, so this entry in the table + is ALWAYS blank and unused. + + */ message StringTable { repeated bytes s = 1; } @@ -159,10 +164,13 @@ message Node { /* Used to densly represent a sequence of nodes that do not have any tags. -We represent these nodes columnwise as four columns: ID's, lats, and -lons, all delta coded and info's. +We represent these nodes columnwise as five columns: ID's, lats, and +lons, all delta coded. And an optional info's column when metadata is not omitted. -Each array corresponds to a column of nodes. */ +We encode keys & vals for all nodes too in one array in the form of ONE array of integers containing key-id and val-id values, using 0 as a delimiter between nodes. + + ( ( )* '0' )* + */ message DenseNodes { repeated sint64 id = 1 [packed = true]; // DELTA coded @@ -170,6 +178,9 @@ message DenseNodes { repeated sint64 lat = 8 [packed = true]; // DELTA coded repeated sint64 lon = 9 [packed = true]; // DELTA coded + + // Special packing of keys and vals into one array. + repeated int32 keys_vals = 10 [packed = true]; }