Support new-format dense nodes that can contain keys and values.

To do this, I need to reserve string id #0 as a delimiter and cannot
generate it anymore.

Forward-compatible format change, in that old files cannot contain
new-format dense nodes, thus the delimiter bug is a non-issue.

src.java/crosby/binary/BinaryParser.java

@@ -26,6 +26,12 @@ public abstract class BinaryParser implements BlockReaderAdapter {
     }
     public static final Date NODATE = new Date();
 
+    /** Get a string based on the index used. 
+     * 
+     * Index 0 is reserved to use as a delimiter, therefore, index 1 corresponds to the first string in the table 
+     * @param id
+     * @return
+     */
     protected String getStringById(int id) {
       return strings[id];
     }

src.java/crosby/binary/StringTable.java

@@ -27,6 +27,12 @@ public class StringTable {
         }
     }
 
+    /** After the stringtable has been built, return the offset of a string in it.
+     * 
+     * Note, value '0' is reserved for use as a delimiter and will not be returned.
+     * @param s
+     * @return
+     */
     public int getIndex(String s) {
         return stringmap.get(s).intValue();
     }
@@ -41,20 +47,21 @@ public class StringTable {
         };
 
         set = counts.keySet().toArray(new String[0]);
-        // Sort based on the frequency.
-        Arrays.sort(set, comparator);
-        // Each group of keys that serializes to the same number of bytes is
-        // sorted lexiconographically.
-        // to maximize deflate compression.
-        Arrays.sort(set, Math.min(0, set.length), Math.min(1 << 7, set.length));
-        Arrays.sort(set, Math.min(1 << 7, set.length), Math.min(1 << 14,
-                set.length));
-        Arrays.sort(set, Math.min(1 << 14, set.length), Math.min(1 << 21,
-                set.length), comparator);
-
+        if (set.length > 0) {
+          // Sort based on the frequency.
+          Arrays.sort(set, comparator);
+          // Each group of keys that serializes to the same number of bytes is
+          // sorted lexiconographically.
+          // to maximize deflate compression.
+          Arrays.sort(set, Math.min(0, set.length-1), Math.min(1 << 7, set.length-1));
+          Arrays.sort(set, Math.min(1 << 7, set.length-1), Math.min(1 << 14,
+              set.length-1));
+          Arrays.sort(set, Math.min(1 << 14, set.length-1), Math.min(1 << 21,
+              set.length-1), comparator);
+        }
         stringmap = new HashMap<String, Integer>(2 * set.length);
         for (int i = 0; i < set.length; i++) {
-            stringmap.put(set[i], new Integer(i));
+            stringmap.put(set[i], new Integer(i+1)); // Index 0 is reserved for use as a delimiter.
         }
         counts = null;
     }
@@ -68,6 +75,7 @@ public class StringTable {
     public Osmformat.StringTable.Builder serialize() {
         Osmformat.StringTable.Builder builder = Osmformat.StringTable
                 .newBuilder();
+        builder.addS(ByteString.copyFromUtf8("")); // Add a unused string at offset 0 which is used as a delimiter.
         for (int i = 0; i < set.length; i++)
             builder.addS(ByteString.copyFromUtf8(set[i]));
         return builder;

src/osmformat.proto

@@ -115,7 +115,12 @@ message PrimitiveGroup {
 }
 
 
-/** String table, contains the common strings in each block */
+/** String table, contains the common strings in each block.
+
+ Note that we reserve index '0' as a delimiter, so this entry in the table
+ is ALWAYS blank and unused.
+
+ */
 message StringTable {
    repeated bytes s = 1;
 }
@@ -159,10 +164,13 @@ message Node {
 
 /* Used to densly represent a sequence of nodes that do not have any tags.
 
-We represent these nodes columnwise as four columns: ID's, lats, and
-lons, all delta coded and info's.
+We represent these nodes columnwise as five columns: ID's, lats, and
+lons, all delta coded. And an optional info's column when metadata is not omitted. 
 
-Each array corresponds to a column of nodes. */
+We encode keys & vals for all nodes too in one array in the form of ONE array of integers containing key-id and val-id values, using 0 as a delimiter between nodes. 
+
+   ( (<keyid> <valid>)* '0' )*
+ */
 message DenseNodes {
    repeated sint64 id = 1 [packed = true]; // DELTA coded
 
@@ -170,6 +178,9 @@ message DenseNodes {
 
    repeated sint64 lat = 8 [packed = true]; // DELTA coded
    repeated sint64 lon = 9 [packed = true]; // DELTA coded
+
+   // Special packing of keys and vals into one array.
+   repeated int32 keys_vals = 10 [packed = true];
 }