From 922411b27af99ab95996c3a0045e8a4b941dc811 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 10:31:24 +0200 Subject: [PATCH 01/13] mockup for outlining tool --- tools/.gitignore | 1 + tools/Makefile | 17 +++++++++++++ tools/osmpbf-outline.cpp | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 tools/.gitignore create mode 100644 tools/Makefile create mode 100644 tools/osmpbf-outline.cpp diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000..3235067 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1 @@ +osmpbf-outline diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..771ef9c --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,17 @@ + +CXX = g++ +CXXFLAGS = -O3 +LDFLAGS = -lpthread +LIB_PROTOBUF = -lz -lprotobuf-lite -losmpbf + +all: osmpbf-outline + +osmpbf-outline: osmpbf-outline.cpp + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIB_PROTOBUF) + +install: + install -m 755 -g root -o root -d $(DESTDIR)/usr/bin + install -m 644 -g root -o root osmpbf-outline $(DESTDIR)/usr/bin + +clean: + rm -f osmpbf-outline diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp new file mode 100644 index 0000000..74dfff2 --- /dev/null +++ b/tools/osmpbf-outline.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include + +bool is_a_tty; + +void msg(const char* format, int color, va_list args) { + if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); + + vfprintf(stderr, format, args); + + if(is_a_tty) fprintf(stderr, "\x1b[0m\n"); + else fprintf(stderr, "\n"); +} + +void err(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 31, args); + va_end(args); + exit(1); +} + +void warn(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 33, args); + va_end(args); +} + +void info(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 37, args); + va_end(args); +} + +int main(int argc, char *argv[]) { + if(isatty(2)) { + is_a_tty = true; + } + + if(argc != 2) + err("usage: %s file.osm.pbf", argv[0]); + + FILE *fp = fopen(argv[1], "r"); + + + fclose(fp); + google::protobuf::ShutdownProtobufLibrary(); +} From e634d7ae1b9c01a33fd0da221b1f450294f46aa1 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 14:35:32 +0200 Subject: [PATCH 02/13] read blobs --- tools/osmpbf-outline.cpp | 78 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 74dfff2..63d7eed 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -1,11 +1,24 @@ #include #include #include +#include #include #include bool is_a_tty; +const int MAX_BLOB_HEADER_SIZE = 64 * 1024; +const int MAX_BLOB_SIZE = 32 * 1024 * 1024; + +char buffer[MAX_BLOB_SIZE]; +char unpack_buffer[MAX_BLOB_SIZE]; + +OSMPBF::BlobHeader blobheader; +OSMPBF::Blob blob; + +bool flag = false; + + void msg(const char* format, int color, va_list args) { if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); @@ -37,6 +50,7 @@ void info(const char* format, ...) { va_end(args); } + int main(int argc, char *argv[]) { if(isatty(2)) { is_a_tty = true; @@ -46,7 +60,71 @@ int main(int argc, char *argv[]) { err("usage: %s file.osm.pbf", argv[0]); FILE *fp = fopen(argv[1], "r"); + while(!feof(fp)) { + int32_t sz; + if(fread(&sz, sizeof(sz), 1, fp) != 1) + break; // EOF + sz = ntohl(sz); + if(sz > MAX_BLOB_HEADER_SIZE) + err("blob-header-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_HEADER_SIZE); + + if(fread(buffer, sz, 1, fp) != 1) + err("unable to read blob-header from file"); + + blobheader.ParseFromArray(buffer, sz); + + info("BlobHeader (%d bytes)", sz); + info(" type = %s", blobheader.type().c_str()); + sz = blobheader.datasize(); + info(" datasize = %u", sz); + if(blobheader.has_indexdata()) + info(" indexdata = %u bytes", blobheader.indexdata().size()); + else + info(" no indexdata"); + + if(sz > MAX_BLOB_SIZE) + err("blob-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_SIZE); + + if(fread(buffer, sz, 1, fp) != 1) + err("unable to read blob from file"); + + blob.ParseFromArray(buffer, sz); + + info("Blob (%d bytes)", sz); + + if(blob.has_raw()) { + flag = true; + info(" contains uncompressed data: %u bytes", blob.raw().size()); + if(blob.raw().size() != blob.raw_size()) + warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + } + + if(blob.has_zlib_data()) { + if(flag) + warn(" contains raw- and zlib-data at the same time"); + + flag = true; + info(" contains zlib-compressed data: %u bytes", blob.zlib_data().size()); + info(" uncompressed size: %u bytes", blob.raw_size()); + } + + if(blob.has_lzma_data()) { + if(flag) + warn(" contains raw- and lzma-data at the same time"); + + flag = true; + info(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + info(" uncompressed size: %u bytes", blob.raw_size()); + + err(" lzma-decompression is not supported"); + } + + if(!flag) + err(" does not contain any known data stream"); + + flag = false; + } fclose(fp); google::protobuf::ShutdownProtobufLibrary(); From d0134bd84077d6248d0765d822cbc751735bbdc9 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 14:41:45 +0200 Subject: [PATCH 03/13] decompression --- tools/osmpbf-outline.cpp | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 63d7eed..b8e3e9e 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -95,9 +95,12 @@ int main(int argc, char *argv[]) { if(blob.has_raw()) { flag = true; - info(" contains uncompressed data: %u bytes", blob.raw().size()); - if(blob.raw().size() != blob.raw_size()) + sz = blob.raw().size(); + info(" contains uncompressed data: %u bytes", sz); + if(sz != blob.raw_size()) warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + + memcpy(unpack_buffer, buffer, sz); } if(blob.has_zlib_data()) { @@ -105,8 +108,29 @@ int main(int argc, char *argv[]) { warn(" contains raw- and zlib-data at the same time"); flag = true; - info(" contains zlib-compressed data: %u bytes", blob.zlib_data().size()); + sz = blob.zlib_data().size(); + info(" contains zlib-compressed data: %u bytes", sz); info(" uncompressed size: %u bytes", blob.raw_size()); + + z_stream z; + + z.next_in = (unsigned char*) blob.zlib_data().c_str(); + z.avail_in = sz; + z.next_out = (unsigned char*) unpack_buffer; + z.avail_out = blob.raw_size(); + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if (inflateInit(&z) != Z_OK) { + err(" failed to init zlib stream"); + } + if (inflate(&z, Z_FINISH) != Z_STREAM_END) { + err(" failed to inflate zlib stream"); + } + if (inflateEnd(&z) != Z_OK) { + err(" failed to deinit zlib stream"); + } } if(blob.has_lzma_data()) { From e82bd8f39c704a758535a0c286be2bbfc344b54a Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 16:35:01 +0200 Subject: [PATCH 04/13] more color, more documentation --- tools/osmpbf-outline.cpp | 135 +++++++++++++++++++++++++++++++-------- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index b8e3e9e..c32b7eb 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -16,9 +16,9 @@ char unpack_buffer[MAX_BLOB_SIZE]; OSMPBF::BlobHeader blobheader; OSMPBF::Blob blob; -bool flag = false; - - +/** + * prints a formatted message to stderr, optionally color coded + */ void msg(const char* format, int color, va_list args) { if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); @@ -28,6 +28,9 @@ void msg(const char* format, int color, va_list args) { else fprintf(stderr, "\n"); } +/** + * prints a formatted message to stderr, color coded to red + */ void err(const char* format, ...) { va_list args; va_start(args, format); @@ -36,6 +39,9 @@ void err(const char* format, ...) { exit(1); } +/** + * prints a formatted message to stderr, color coded to yellow + */ void warn(const char* format, ...) { va_list args; va_start(args, format); @@ -43,113 +49,188 @@ void warn(const char* format, ...) { va_end(args); } +/** + * prints a formatted message to stderr, color coded to green + */ void info(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 32, args); + va_end(args); +} + +/** + * prints a formatted message to stderr, color coded to white + */ +void debug(const char* format, ...) { va_list args; va_start(args, format); msg(format, 37, args); va_end(args); } - +/** + * application main method + */ int main(int argc, char *argv[]) { - if(isatty(2)) { - is_a_tty = true; - } + // check if stderr is a tty and set a global bool + is_a_tty = (bool)isatty(2); + // check for proper command line args if(argc != 2) err("usage: %s file.osm.pbf", argv[0]); + // open specified file FILE *fp = fopen(argv[1], "r"); - while(!feof(fp)) { - int32_t sz; - if(fread(&sz, sizeof(sz), 1, fp) != 1) - break; // EOF + // read while the file has not reached its end + while(!feof(fp)) { + // storage of size, used multiple times + int32_t sz; + + // boolean flag, used multiple times + bool flag = false; + + // read the first 4 bytes of the file, this is the size of the blob-header + if(fread(&sz, sizeof(sz), 1, fp) != 1) + break; // end of file reached + + // convert the size from network byte-order to host byte-order sz = ntohl(sz); + + // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE if(sz > MAX_BLOB_HEADER_SIZE) err("blob-header-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_HEADER_SIZE); + // read the blob-header from the file if(fread(buffer, sz, 1, fp) != 1) err("unable to read blob-header from file"); + // parse the blob-header from the read-buffer blobheader.ParseFromArray(buffer, sz); + // tell about the blob-header info("BlobHeader (%d bytes)", sz); - info(" type = %s", blobheader.type().c_str()); - sz = blobheader.datasize(); - info(" datasize = %u", sz); - if(blobheader.has_indexdata()) - info(" indexdata = %u bytes", blobheader.indexdata().size()); - else - info(" no indexdata"); + debug(" type = %s", blobheader.type().c_str()); + // size of the following blob + sz = blobheader.datasize(); + debug(" datasize = %u", sz); + + // optional indexdata + if(blobheader.has_indexdata()) + debug(" indexdata = %u bytes", blobheader.indexdata().size()); + else + debug(" no indexdata"); + + // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > MAX_BLOB_SIZE) err("blob-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_SIZE); + // read the blob from the file if(fread(buffer, sz, 1, fp) != 1) err("unable to read blob from file"); + // parse the blob from the read-buffer blob.ParseFromArray(buffer, sz); + // tell about the blob-header info("Blob (%d bytes)", sz); + // if the blob has uncompressed data if(blob.has_raw()) { + // raise the flag - we have at least one datastream flag = true; + + // size of the blob-data sz = blob.raw().size(); - info(" contains uncompressed data: %u bytes", sz); + + // check that raw_size is set correctly if(sz != blob.raw_size()) warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + // tell about the blob-data + debug(" contains uncompressed data: %u bytes", sz); + + // copy the uncompressed data over to the unpack_buffer memcpy(unpack_buffer, buffer, sz); } + // if the blob has zlib-compressed data if(blob.has_zlib_data()) { + // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) warn(" contains raw- and zlib-data at the same time"); + // raise the flag - we have at least one datastream flag = true; - sz = blob.zlib_data().size(); - info(" contains zlib-compressed data: %u bytes", sz); - info(" uncompressed size: %u bytes", blob.raw_size()); + // the size of the compressesd data + sz = blob.zlib_data().size(); + + // tell about the compressed data + debug(" contains zlib-compressed data: %u bytes", sz); + debug(" uncompressed size: %u bytes", blob.raw_size()); + + // zlib information z_stream z; + // next byte to decompress z.next_in = (unsigned char*) blob.zlib_data().c_str(); + + // number of bytes to decompress z.avail_in = sz; + + // place of next decompressed byte z.next_out = (unsigned char*) unpack_buffer; + + // space for decompressed data z.avail_out = blob.raw_size(); + + // misc z.zalloc = Z_NULL; z.zfree = Z_NULL; z.opaque = Z_NULL; - if (inflateInit(&z) != Z_OK) { + if(inflateInit(&z) != Z_OK) { err(" failed to init zlib stream"); } - if (inflate(&z, Z_FINISH) != Z_STREAM_END) { + if(inflate(&z, Z_FINISH) != Z_STREAM_END) { err(" failed to inflate zlib stream"); } - if (inflateEnd(&z) != Z_OK) { + if(inflateEnd(&z) != Z_OK) { err(" failed to deinit zlib stream"); } } + // if the blob has lzma-compressed data if(blob.has_lzma_data()) { + // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) warn(" contains raw- and lzma-data at the same time"); + // raise the flag - we have at least one datastream flag = true; - info(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); - info(" uncompressed size: %u bytes", blob.raw_size()); + // tell about the compressed data + debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + debug(" uncompressed size: %u bytes", blob.raw_size()); + + // issue a warning, lzma compression is not yet supported err(" lzma-decompression is not supported"); } + // check the flag is raised and we have at least one data-stream if(!flag) err(" does not contain any known data stream"); + // lower the flag flag = false; } + // close the file pointer fclose(fp); + + // clean up the protobuf lib google::protobuf::ShutdownProtobufLibrary(); } From a02882c42338fe45982fab6874f1998e007129fa Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 16:46:09 +0200 Subject: [PATCH 05/13] read osmheader --- tools/osmpbf-outline.cpp | 71 ++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index c32b7eb..352c9f9 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -15,6 +15,7 @@ char unpack_buffer[MAX_BLOB_SIZE]; OSMPBF::BlobHeader blobheader; OSMPBF::Blob blob; +OSMPBF::HeaderBlock osmheader; /** * prints a formatted message to stderr, optionally color coded @@ -111,17 +112,17 @@ int main(int argc, char *argv[]) { // tell about the blob-header info("BlobHeader (%d bytes)", sz); - debug(" type = %s", blobheader.type().c_str()); + debug(" type = %s", blobheader.type().c_str()); // size of the following blob sz = blobheader.datasize(); - debug(" datasize = %u", sz); + debug(" datasize = %u", sz); // optional indexdata if(blobheader.has_indexdata()) - debug(" indexdata = %u bytes", blobheader.indexdata().size()); + debug(" indexdata = %u bytes", blobheader.indexdata().size()); else - debug(" no indexdata"); + debug(" no indexdata"); // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > MAX_BLOB_SIZE) @@ -147,10 +148,10 @@ int main(int argc, char *argv[]) { // check that raw_size is set correctly if(sz != blob.raw_size()) - warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + warn(" reports wrong raw_size: %u bytes", blob.raw_size()); // tell about the blob-data - debug(" contains uncompressed data: %u bytes", sz); + debug(" contains uncompressed data: %u bytes", sz); // copy the uncompressed data over to the unpack_buffer memcpy(unpack_buffer, buffer, sz); @@ -160,7 +161,7 @@ int main(int argc, char *argv[]) { if(blob.has_zlib_data()) { // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) - warn(" contains raw- and zlib-data at the same time"); + warn(" contains raw- and zlib-data at the same time"); // raise the flag - we have at least one datastream flag = true; @@ -169,8 +170,8 @@ int main(int argc, char *argv[]) { sz = blob.zlib_data().size(); // tell about the compressed data - debug(" contains zlib-compressed data: %u bytes", sz); - debug(" uncompressed size: %u bytes", blob.raw_size()); + debug(" contains zlib-compressed data: %u bytes", sz); + debug(" uncompressed size: %u bytes", blob.raw_size()); // zlib information z_stream z; @@ -193,39 +194,75 @@ int main(int argc, char *argv[]) { z.opaque = Z_NULL; if(inflateInit(&z) != Z_OK) { - err(" failed to init zlib stream"); + err(" failed to init zlib stream"); } if(inflate(&z, Z_FINISH) != Z_STREAM_END) { - err(" failed to inflate zlib stream"); + err(" failed to inflate zlib stream"); } if(inflateEnd(&z) != Z_OK) { - err(" failed to deinit zlib stream"); + err(" failed to deinit zlib stream"); } + + // unpacked size + sz = z.total_out; } // if the blob has lzma-compressed data if(blob.has_lzma_data()) { // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) - warn(" contains raw- and lzma-data at the same time"); + warn(" contains raw- and lzma-data at the same time"); // raise the flag - we have at least one datastream flag = true; // tell about the compressed data - debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); - debug(" uncompressed size: %u bytes", blob.raw_size()); + debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + debug(" uncompressed size: %u bytes", blob.raw_size()); // issue a warning, lzma compression is not yet supported - err(" lzma-decompression is not supported"); + err(" lzma-decompression is not supported"); } // check the flag is raised and we have at least one data-stream if(!flag) - err(" does not contain any known data stream"); + err(" does not contain any known data stream"); // lower the flag flag = false; + + // switch between different blob-types + if(blobheader.type() == "OSMHeader") { + // tell about the OSMHeader blob + info(" OSMHeader"); + + // parse the OSMHeader from the blob + osmheader.ParseFromArray(unpack_buffer, sz); + + // tell about the required features + for(int i = 0, l = osmheader.required_features_size(); i < l; i++) + debug(" required_feature: %s", osmheader.required_features(i).c_str()); + + // tell about the optional features + for(int i = 0, l = osmheader.optional_features_size(); i < l; i++) + debug(" required_feature: %s", osmheader.optional_features(i).c_str()); + + // tell about the writing program + debug(" writingprogram: %s", osmheader.writingprogram().c_str()); + + // tell about the source + debug(" source: %s", osmheader.source().c_str()); + } + + else if(blobheader.type() == "OSMData") { + // tell about the OSMData blob + info(" OSMData"); + } + + else { + // unknown blob type + warn(" unknown blob type: %s", blobheader.type().c_str()); + } } // close the file pointer From 072daaee0ad49f2c6c78b9cdab59897be85631e2 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:35:13 +0200 Subject: [PATCH 06/13] always color to stdout --- tools/osmpbf-outline.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 352c9f9..b47f004 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -5,8 +5,6 @@ #include #include -bool is_a_tty; - const int MAX_BLOB_HEADER_SIZE = 64 * 1024; const int MAX_BLOB_SIZE = 32 * 1024 * 1024; @@ -21,12 +19,9 @@ OSMPBF::HeaderBlock osmheader; * prints a formatted message to stderr, optionally color coded */ void msg(const char* format, int color, va_list args) { - if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); - - vfprintf(stderr, format, args); - - if(is_a_tty) fprintf(stderr, "\x1b[0m\n"); - else fprintf(stderr, "\n"); + fprintf(stdout, "\x1b[0;%dm", color); + vfprintf(stdout, format, args); + fprintf(stdout, "\x1b[0m\n"); } /** @@ -74,9 +69,6 @@ void debug(const char* format, ...) { * application main method */ int main(int argc, char *argv[]) { - // check if stderr is a tty and set a global bool - is_a_tty = (bool)isatty(2); - // check for proper command line args if(argc != 2) err("usage: %s file.osm.pbf", argv[0]); From f5a5dde412c26dd551d3527c85064433629a71e3 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:35:40 +0200 Subject: [PATCH 07/13] stdout, i said! --- tools/osmpbf-outline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index b47f004..e265c9c 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -16,7 +16,7 @@ OSMPBF::Blob blob; OSMPBF::HeaderBlock osmheader; /** - * prints a formatted message to stderr, optionally color coded + * prints a formatted message to stdout, optionally color coded */ void msg(const char* format, int color, va_list args) { fprintf(stdout, "\x1b[0;%dm", color); From 3727704d2bc877a9b4c61d9e61006b0eb128661d Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:44:54 +0200 Subject: [PATCH 08/13] even more documentation --- tools/osmpbf-outline.cpp | 48 +++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index e265c9c..2373b00 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -1,32 +1,50 @@ +// used for va_list in debug-print methods #include + +// file io lib #include + +// zlib compression is used inside the pbf blobs #include + +// netinet provides the network-byte-order conversion function #include + +// this is the header to libosmpbf for reading and writing the lowlevel blob storags #include + +// this is the header to libosmpbf for reading and writing the highlevel osm objects #include +// the maximum size of a blob-header in bytes const int MAX_BLOB_HEADER_SIZE = 64 * 1024; + +// the maximum size of a blob in bytes const int MAX_BLOB_SIZE = 32 * 1024 * 1024; +// buffer for reading a compressed blob from file char buffer[MAX_BLOB_SIZE]; + +// buffer for decompressing the blob char unpack_buffer[MAX_BLOB_SIZE]; +// pbf structur of a BlobHeader OSMPBF::BlobHeader blobheader; + +// pbf structur of a Blob OSMPBF::Blob blob; + +// pbf structur of a OSM HeaderBlock OSMPBF::HeaderBlock osmheader; -/** - * prints a formatted message to stdout, optionally color coded - */ +// prints a formatted message to stdout, optionally color coded void msg(const char* format, int color, va_list args) { fprintf(stdout, "\x1b[0;%dm", color); vfprintf(stdout, format, args); fprintf(stdout, "\x1b[0m\n"); } -/** - * prints a formatted message to stderr, color coded to red - */ +// prints a formatted message to stderr, color coded to red void err(const char* format, ...) { va_list args; va_start(args, format); @@ -35,9 +53,7 @@ void err(const char* format, ...) { exit(1); } -/** - * prints a formatted message to stderr, color coded to yellow - */ +// prints a formatted message to stderr, color coded to yellow void warn(const char* format, ...) { va_list args; va_start(args, format); @@ -45,9 +61,7 @@ void warn(const char* format, ...) { va_end(args); } -/** - * prints a formatted message to stderr, color coded to green - */ +// prints a formatted message to stderr, color coded to green void info(const char* format, ...) { va_list args; va_start(args, format); @@ -55,9 +69,7 @@ void info(const char* format, ...) { va_end(args); } -/** - * prints a formatted message to stderr, color coded to white - */ +// prints a formatted message to stderr, color coded to white void debug(const char* format, ...) { va_list args; va_start(args, format); @@ -65,9 +77,7 @@ void debug(const char* format, ...) { va_end(args); } -/** - * application main method - */ +// application main method int main(int argc, char *argv[]) { // check for proper command line args if(argc != 2) @@ -113,8 +123,6 @@ int main(int argc, char *argv[]) { // optional indexdata if(blobheader.has_indexdata()) debug(" indexdata = %u bytes", blobheader.indexdata().size()); - else - debug(" no indexdata"); // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > MAX_BLOB_SIZE) From f1c905e60f326e2280967e8ce9de01e351e88b53 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:51:59 +0200 Subject: [PATCH 09/13] bbox reader --- tools/osmpbf-outline.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 2373b00..5d3d00a 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -22,6 +22,9 @@ const int MAX_BLOB_HEADER_SIZE = 64 * 1024; // the maximum size of a blob in bytes const int MAX_BLOB_SIZE = 32 * 1024 * 1024; +// nanodegree multiplier +static const long int NANO = 1000 * 1000 * 1000; + // buffer for reading a compressed blob from file char buffer[MAX_BLOB_SIZE]; @@ -239,6 +242,14 @@ int main(int argc, char *argv[]) { // parse the OSMHeader from the blob osmheader.ParseFromArray(unpack_buffer, sz); + // tell about the bbox + if(osmheader.has_bbox()) { + OSMPBF::HeaderBBox bbox = osmheader.bbox(); + debug(" bbox: %.7f,%.7f,%.7f,%.7f", + (double)bbox.left() / NANO, (double)bbox.bottom() / NANO, + (double)bbox.right() / NANO, (double)bbox.top() / NANO); + } + // tell about the required features for(int i = 0, l = osmheader.required_features_size(); i < l; i++) debug(" required_feature: %s", osmheader.required_features(i).c_str()); From 0fbe0f8cff1e624a7989bc61493dd65728bfd00c Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:54:19 +0200 Subject: [PATCH 10/13] make source & writingprogram optional --- tools/osmpbf-outline.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 5d3d00a..b2089ee 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -259,10 +259,12 @@ int main(int argc, char *argv[]) { debug(" required_feature: %s", osmheader.optional_features(i).c_str()); // tell about the writing program - debug(" writingprogram: %s", osmheader.writingprogram().c_str()); + if(osmheader.has_writingprogram()); + debug(" writingprogram: %s", osmheader.writingprogram().c_str()); // tell about the source - debug(" source: %s", osmheader.source().c_str()); + if(osmheader.has_source()) + debug(" source: %s", osmheader.source().c_str()); } else if(blobheader.type() == "OSMData") { From e2d430cbf947a373b0e94e26c43e59ac8f4c90f8 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 22:17:29 +0200 Subject: [PATCH 11/13] look into the block --- tools/osmpbf-outline.cpp | 86 ++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index b2089ee..3d303c7 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -31,14 +31,17 @@ char buffer[MAX_BLOB_SIZE]; // buffer for decompressing the blob char unpack_buffer[MAX_BLOB_SIZE]; -// pbf structur of a BlobHeader +// pbf struct of a BlobHeader OSMPBF::BlobHeader blobheader; -// pbf structur of a Blob +// pbf struct of a Blob OSMPBF::Blob blob; -// pbf structur of a OSM HeaderBlock -OSMPBF::HeaderBlock osmheader; +// pbf struct of an OSM HeaderBlock +OSMPBF::HeaderBlock headerblock; + +// pbf struct of an OSM PrimitiveBlock +OSMPBF::PrimitiveBlock primblock; // prints a formatted message to stdout, optionally color coded void msg(const char* format, int color, va_list args) { @@ -239,37 +242,87 @@ int main(int argc, char *argv[]) { // tell about the OSMHeader blob info(" OSMHeader"); - // parse the OSMHeader from the blob - osmheader.ParseFromArray(unpack_buffer, sz); + // parse the HeaderBlock from the blob + headerblock.ParseFromArray(unpack_buffer, sz); // tell about the bbox - if(osmheader.has_bbox()) { - OSMPBF::HeaderBBox bbox = osmheader.bbox(); + if(headerblock.has_bbox()) { + OSMPBF::HeaderBBox bbox = headerblock.bbox(); debug(" bbox: %.7f,%.7f,%.7f,%.7f", (double)bbox.left() / NANO, (double)bbox.bottom() / NANO, (double)bbox.right() / NANO, (double)bbox.top() / NANO); } // tell about the required features - for(int i = 0, l = osmheader.required_features_size(); i < l; i++) - debug(" required_feature: %s", osmheader.required_features(i).c_str()); + for(int i = 0, l = headerblock.required_features_size(); i < l; i++) + debug(" required_feature: %s", headerblock.required_features(i).c_str()); // tell about the optional features - for(int i = 0, l = osmheader.optional_features_size(); i < l; i++) - debug(" required_feature: %s", osmheader.optional_features(i).c_str()); + for(int i = 0, l = headerblock.optional_features_size(); i < l; i++) + debug(" required_feature: %s", headerblock.optional_features(i).c_str()); // tell about the writing program - if(osmheader.has_writingprogram()); - debug(" writingprogram: %s", osmheader.writingprogram().c_str()); + if(headerblock.has_writingprogram()); + debug(" writingprogram: %s", headerblock.writingprogram().c_str()); // tell about the source - if(osmheader.has_source()) - debug(" source: %s", osmheader.source().c_str()); + if(headerblock.has_source()) + debug(" source: %s", headerblock.source().c_str()); } else if(blobheader.type() == "OSMData") { // tell about the OSMData blob info(" OSMData"); + + // parse the PrimitiveBlock from the blob + primblock.ParseFromArray(unpack_buffer, sz); + + // tell about the stringtable + debug(" stringtable: %u items", primblock.stringtable().s_size()); + + // number of PrimitiveGroups + debug(" primitivegroups: %u groups", primblock.primitivegroup_size()); + + // iterate over all PrimitiveGroups + for(int i = 0, l = primblock.primitivegroup_size(); i < l; i++) { + // one PrimitiveGroup from the the Block + OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i); + + // tell about nodes + if(pg.nodes_size() > 0) { + // raise the flag - we have at least one item type + flag = true; + + debug(" nodes: %d", pg.nodes_size()); + } + + // tell about densenodes + if(pg.has_dense()) { + // raise the flag - we have at leastone item type + flag = true; + + debug(" dense nodes: %d", pg.dense().id_size()); + } + + // tell about ways + if(pg.ways_size() > 0) { + // raise the flag - we have at leastone item type + flag = true; + + debug(" ways: %d", pg.ways_size()); + } + + // tell aboutrelations + if(pg.relations_size() > 0) { + // raise the flag - we have at leastone item type + flag = true; + + debug(" relations: %d", pg.relations_size()); + } + + if(!flag) + warn(" contains no items"); + } } else { @@ -284,3 +337,4 @@ int main(int argc, char *argv[]) { // clean up the protobuf lib google::protobuf::ShutdownProtobufLibrary(); } + From a1f2b8133251e4a848449ca15b1ccc951e57fb17 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 22:57:29 +0200 Subject: [PATCH 12/13] tell about meta-info --- tools/osmpbf-outline.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 3d303c7..f032d3b 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -294,6 +294,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" nodes: %d", pg.nodes_size()); + if(pg.nodes(0).has_info()) + debug(" with meta-info"); } // tell about densenodes @@ -302,6 +304,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" dense nodes: %d", pg.dense().id_size()); + if(pg.dense().has_denseinfo()) + debug(" with meta-info"); } // tell about ways @@ -310,6 +314,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" ways: %d", pg.ways_size()); + if(pg.ways(0).has_info()) + debug(" with meta-info"); } // tell aboutrelations @@ -318,6 +324,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" relations: %d", pg.relations_size()); + if(pg.relations(0).has_info()) + debug(" with meta-info"); } if(!flag) From b6cf2110eca09d0a30ed80409c084b7fcdcab853 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 25 May 2011 21:28:43 +0200 Subject: [PATCH 13/13] show primblock metainfo --- tools/osmpbf-outline.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index f032d3b..af357d5 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -277,6 +277,12 @@ int main(int argc, char *argv[]) { // parse the PrimitiveBlock from the blob primblock.ParseFromArray(unpack_buffer, sz); + // tell about the block's meta info + debug(" granularity: %u", primblock.granularity()); + debug(" lat_offset: %u", primblock.lat_offset()); + debug(" lon_offset: %u", primblock.lon_offset()); + debug(" date_granularity: %u", primblock.date_granularity()); + // tell about the stringtable debug(" stringtable: %u items", primblock.stringtable().s_size());