From 922411b27af99ab95996c3a0045e8a4b941dc811 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 10:31:24 +0200 Subject: [PATCH 01/29] mockup for outlining tool --- tools/.gitignore | 1 + tools/Makefile | 17 +++++++++++++ tools/osmpbf-outline.cpp | 53 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 tools/.gitignore create mode 100644 tools/Makefile create mode 100644 tools/osmpbf-outline.cpp diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000..3235067 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1 @@ +osmpbf-outline diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..771ef9c --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,17 @@ + +CXX = g++ +CXXFLAGS = -O3 +LDFLAGS = -lpthread +LIB_PROTOBUF = -lz -lprotobuf-lite -losmpbf + +all: osmpbf-outline + +osmpbf-outline: osmpbf-outline.cpp + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIB_PROTOBUF) + +install: + install -m 755 -g root -o root -d $(DESTDIR)/usr/bin + install -m 644 -g root -o root osmpbf-outline $(DESTDIR)/usr/bin + +clean: + rm -f osmpbf-outline diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp new file mode 100644 index 0000000..74dfff2 --- /dev/null +++ b/tools/osmpbf-outline.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include + +bool is_a_tty; + +void msg(const char* format, int color, va_list args) { + if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); + + vfprintf(stderr, format, args); + + if(is_a_tty) fprintf(stderr, "\x1b[0m\n"); + else fprintf(stderr, "\n"); +} + +void err(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 31, args); + va_end(args); + exit(1); +} + +void warn(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 33, args); + va_end(args); +} + +void info(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 37, args); + va_end(args); +} + +int main(int argc, char *argv[]) { + if(isatty(2)) { + is_a_tty = true; + } + + if(argc != 2) + err("usage: %s file.osm.pbf", argv[0]); + + FILE *fp = fopen(argv[1], "r"); + + + fclose(fp); + google::protobuf::ShutdownProtobufLibrary(); +} From e634d7ae1b9c01a33fd0da221b1f450294f46aa1 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 14:35:32 +0200 Subject: [PATCH 02/29] read blobs --- tools/osmpbf-outline.cpp | 78 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 74dfff2..63d7eed 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -1,11 +1,24 @@ #include #include #include +#include #include #include bool is_a_tty; +const int MAX_BLOB_HEADER_SIZE = 64 * 1024; +const int MAX_BLOB_SIZE = 32 * 1024 * 1024; + +char buffer[MAX_BLOB_SIZE]; +char unpack_buffer[MAX_BLOB_SIZE]; + +OSMPBF::BlobHeader blobheader; +OSMPBF::Blob blob; + +bool flag = false; + + void msg(const char* format, int color, va_list args) { if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); @@ -37,6 +50,7 @@ void info(const char* format, ...) { va_end(args); } + int main(int argc, char *argv[]) { if(isatty(2)) { is_a_tty = true; @@ -46,7 +60,71 @@ int main(int argc, char *argv[]) { err("usage: %s file.osm.pbf", argv[0]); FILE *fp = fopen(argv[1], "r"); + while(!feof(fp)) { + int32_t sz; + if(fread(&sz, sizeof(sz), 1, fp) != 1) + break; // EOF + sz = ntohl(sz); + if(sz > MAX_BLOB_HEADER_SIZE) + err("blob-header-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_HEADER_SIZE); + + if(fread(buffer, sz, 1, fp) != 1) + err("unable to read blob-header from file"); + + blobheader.ParseFromArray(buffer, sz); + + info("BlobHeader (%d bytes)", sz); + info(" type = %s", blobheader.type().c_str()); + sz = blobheader.datasize(); + info(" datasize = %u", sz); + if(blobheader.has_indexdata()) + info(" indexdata = %u bytes", blobheader.indexdata().size()); + else + info(" no indexdata"); + + if(sz > MAX_BLOB_SIZE) + err("blob-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_SIZE); + + if(fread(buffer, sz, 1, fp) != 1) + err("unable to read blob from file"); + + blob.ParseFromArray(buffer, sz); + + info("Blob (%d bytes)", sz); + + if(blob.has_raw()) { + flag = true; + info(" contains uncompressed data: %u bytes", blob.raw().size()); + if(blob.raw().size() != blob.raw_size()) + warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + } + + if(blob.has_zlib_data()) { + if(flag) + warn(" contains raw- and zlib-data at the same time"); + + flag = true; + info(" contains zlib-compressed data: %u bytes", blob.zlib_data().size()); + info(" uncompressed size: %u bytes", blob.raw_size()); + } + + if(blob.has_lzma_data()) { + if(flag) + warn(" contains raw- and lzma-data at the same time"); + + flag = true; + info(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + info(" uncompressed size: %u bytes", blob.raw_size()); + + err(" lzma-decompression is not supported"); + } + + if(!flag) + err(" does not contain any known data stream"); + + flag = false; + } fclose(fp); google::protobuf::ShutdownProtobufLibrary(); From d0134bd84077d6248d0765d822cbc751735bbdc9 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 14:41:45 +0200 Subject: [PATCH 03/29] decompression --- tools/osmpbf-outline.cpp | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 63d7eed..b8e3e9e 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -95,9 +95,12 @@ int main(int argc, char *argv[]) { if(blob.has_raw()) { flag = true; - info(" contains uncompressed data: %u bytes", blob.raw().size()); - if(blob.raw().size() != blob.raw_size()) + sz = blob.raw().size(); + info(" contains uncompressed data: %u bytes", sz); + if(sz != blob.raw_size()) warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + + memcpy(unpack_buffer, buffer, sz); } if(blob.has_zlib_data()) { @@ -105,8 +108,29 @@ int main(int argc, char *argv[]) { warn(" contains raw- and zlib-data at the same time"); flag = true; - info(" contains zlib-compressed data: %u bytes", blob.zlib_data().size()); + sz = blob.zlib_data().size(); + info(" contains zlib-compressed data: %u bytes", sz); info(" uncompressed size: %u bytes", blob.raw_size()); + + z_stream z; + + z.next_in = (unsigned char*) blob.zlib_data().c_str(); + z.avail_in = sz; + z.next_out = (unsigned char*) unpack_buffer; + z.avail_out = blob.raw_size(); + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if (inflateInit(&z) != Z_OK) { + err(" failed to init zlib stream"); + } + if (inflate(&z, Z_FINISH) != Z_STREAM_END) { + err(" failed to inflate zlib stream"); + } + if (inflateEnd(&z) != Z_OK) { + err(" failed to deinit zlib stream"); + } } if(blob.has_lzma_data()) { From e82bd8f39c704a758535a0c286be2bbfc344b54a Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 16:35:01 +0200 Subject: [PATCH 04/29] more color, more documentation --- tools/osmpbf-outline.cpp | 135 +++++++++++++++++++++++++++++++-------- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index b8e3e9e..c32b7eb 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -16,9 +16,9 @@ char unpack_buffer[MAX_BLOB_SIZE]; OSMPBF::BlobHeader blobheader; OSMPBF::Blob blob; -bool flag = false; - - +/** + * prints a formatted message to stderr, optionally color coded + */ void msg(const char* format, int color, va_list args) { if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); @@ -28,6 +28,9 @@ void msg(const char* format, int color, va_list args) { else fprintf(stderr, "\n"); } +/** + * prints a formatted message to stderr, color coded to red + */ void err(const char* format, ...) { va_list args; va_start(args, format); @@ -36,6 +39,9 @@ void err(const char* format, ...) { exit(1); } +/** + * prints a formatted message to stderr, color coded to yellow + */ void warn(const char* format, ...) { va_list args; va_start(args, format); @@ -43,113 +49,188 @@ void warn(const char* format, ...) { va_end(args); } +/** + * prints a formatted message to stderr, color coded to green + */ void info(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 32, args); + va_end(args); +} + +/** + * prints a formatted message to stderr, color coded to white + */ +void debug(const char* format, ...) { va_list args; va_start(args, format); msg(format, 37, args); va_end(args); } - +/** + * application main method + */ int main(int argc, char *argv[]) { - if(isatty(2)) { - is_a_tty = true; - } + // check if stderr is a tty and set a global bool + is_a_tty = (bool)isatty(2); + // check for proper command line args if(argc != 2) err("usage: %s file.osm.pbf", argv[0]); + // open specified file FILE *fp = fopen(argv[1], "r"); - while(!feof(fp)) { - int32_t sz; - if(fread(&sz, sizeof(sz), 1, fp) != 1) - break; // EOF + // read while the file has not reached its end + while(!feof(fp)) { + // storage of size, used multiple times + int32_t sz; + + // boolean flag, used multiple times + bool flag = false; + + // read the first 4 bytes of the file, this is the size of the blob-header + if(fread(&sz, sizeof(sz), 1, fp) != 1) + break; // end of file reached + + // convert the size from network byte-order to host byte-order sz = ntohl(sz); + + // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE if(sz > MAX_BLOB_HEADER_SIZE) err("blob-header-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_HEADER_SIZE); + // read the blob-header from the file if(fread(buffer, sz, 1, fp) != 1) err("unable to read blob-header from file"); + // parse the blob-header from the read-buffer blobheader.ParseFromArray(buffer, sz); + // tell about the blob-header info("BlobHeader (%d bytes)", sz); - info(" type = %s", blobheader.type().c_str()); - sz = blobheader.datasize(); - info(" datasize = %u", sz); - if(blobheader.has_indexdata()) - info(" indexdata = %u bytes", blobheader.indexdata().size()); - else - info(" no indexdata"); + debug(" type = %s", blobheader.type().c_str()); + // size of the following blob + sz = blobheader.datasize(); + debug(" datasize = %u", sz); + + // optional indexdata + if(blobheader.has_indexdata()) + debug(" indexdata = %u bytes", blobheader.indexdata().size()); + else + debug(" no indexdata"); + + // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > MAX_BLOB_SIZE) err("blob-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_SIZE); + // read the blob from the file if(fread(buffer, sz, 1, fp) != 1) err("unable to read blob from file"); + // parse the blob from the read-buffer blob.ParseFromArray(buffer, sz); + // tell about the blob-header info("Blob (%d bytes)", sz); + // if the blob has uncompressed data if(blob.has_raw()) { + // raise the flag - we have at least one datastream flag = true; + + // size of the blob-data sz = blob.raw().size(); - info(" contains uncompressed data: %u bytes", sz); + + // check that raw_size is set correctly if(sz != blob.raw_size()) warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + // tell about the blob-data + debug(" contains uncompressed data: %u bytes", sz); + + // copy the uncompressed data over to the unpack_buffer memcpy(unpack_buffer, buffer, sz); } + // if the blob has zlib-compressed data if(blob.has_zlib_data()) { + // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) warn(" contains raw- and zlib-data at the same time"); + // raise the flag - we have at least one datastream flag = true; - sz = blob.zlib_data().size(); - info(" contains zlib-compressed data: %u bytes", sz); - info(" uncompressed size: %u bytes", blob.raw_size()); + // the size of the compressesd data + sz = blob.zlib_data().size(); + + // tell about the compressed data + debug(" contains zlib-compressed data: %u bytes", sz); + debug(" uncompressed size: %u bytes", blob.raw_size()); + + // zlib information z_stream z; + // next byte to decompress z.next_in = (unsigned char*) blob.zlib_data().c_str(); + + // number of bytes to decompress z.avail_in = sz; + + // place of next decompressed byte z.next_out = (unsigned char*) unpack_buffer; + + // space for decompressed data z.avail_out = blob.raw_size(); + + // misc z.zalloc = Z_NULL; z.zfree = Z_NULL; z.opaque = Z_NULL; - if (inflateInit(&z) != Z_OK) { + if(inflateInit(&z) != Z_OK) { err(" failed to init zlib stream"); } - if (inflate(&z, Z_FINISH) != Z_STREAM_END) { + if(inflate(&z, Z_FINISH) != Z_STREAM_END) { err(" failed to inflate zlib stream"); } - if (inflateEnd(&z) != Z_OK) { + if(inflateEnd(&z) != Z_OK) { err(" failed to deinit zlib stream"); } } + // if the blob has lzma-compressed data if(blob.has_lzma_data()) { + // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) warn(" contains raw- and lzma-data at the same time"); + // raise the flag - we have at least one datastream flag = true; - info(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); - info(" uncompressed size: %u bytes", blob.raw_size()); + // tell about the compressed data + debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + debug(" uncompressed size: %u bytes", blob.raw_size()); + + // issue a warning, lzma compression is not yet supported err(" lzma-decompression is not supported"); } + // check the flag is raised and we have at least one data-stream if(!flag) err(" does not contain any known data stream"); + // lower the flag flag = false; } + // close the file pointer fclose(fp); + + // clean up the protobuf lib google::protobuf::ShutdownProtobufLibrary(); } From a02882c42338fe45982fab6874f1998e007129fa Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 16:46:09 +0200 Subject: [PATCH 05/29] read osmheader --- tools/osmpbf-outline.cpp | 71 ++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index c32b7eb..352c9f9 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -15,6 +15,7 @@ char unpack_buffer[MAX_BLOB_SIZE]; OSMPBF::BlobHeader blobheader; OSMPBF::Blob blob; +OSMPBF::HeaderBlock osmheader; /** * prints a formatted message to stderr, optionally color coded @@ -111,17 +112,17 @@ int main(int argc, char *argv[]) { // tell about the blob-header info("BlobHeader (%d bytes)", sz); - debug(" type = %s", blobheader.type().c_str()); + debug(" type = %s", blobheader.type().c_str()); // size of the following blob sz = blobheader.datasize(); - debug(" datasize = %u", sz); + debug(" datasize = %u", sz); // optional indexdata if(blobheader.has_indexdata()) - debug(" indexdata = %u bytes", blobheader.indexdata().size()); + debug(" indexdata = %u bytes", blobheader.indexdata().size()); else - debug(" no indexdata"); + debug(" no indexdata"); // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > MAX_BLOB_SIZE) @@ -147,10 +148,10 @@ int main(int argc, char *argv[]) { // check that raw_size is set correctly if(sz != blob.raw_size()) - warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + warn(" reports wrong raw_size: %u bytes", blob.raw_size()); // tell about the blob-data - debug(" contains uncompressed data: %u bytes", sz); + debug(" contains uncompressed data: %u bytes", sz); // copy the uncompressed data over to the unpack_buffer memcpy(unpack_buffer, buffer, sz); @@ -160,7 +161,7 @@ int main(int argc, char *argv[]) { if(blob.has_zlib_data()) { // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) - warn(" contains raw- and zlib-data at the same time"); + warn(" contains raw- and zlib-data at the same time"); // raise the flag - we have at least one datastream flag = true; @@ -169,8 +170,8 @@ int main(int argc, char *argv[]) { sz = blob.zlib_data().size(); // tell about the compressed data - debug(" contains zlib-compressed data: %u bytes", sz); - debug(" uncompressed size: %u bytes", blob.raw_size()); + debug(" contains zlib-compressed data: %u bytes", sz); + debug(" uncompressed size: %u bytes", blob.raw_size()); // zlib information z_stream z; @@ -193,39 +194,75 @@ int main(int argc, char *argv[]) { z.opaque = Z_NULL; if(inflateInit(&z) != Z_OK) { - err(" failed to init zlib stream"); + err(" failed to init zlib stream"); } if(inflate(&z, Z_FINISH) != Z_STREAM_END) { - err(" failed to inflate zlib stream"); + err(" failed to inflate zlib stream"); } if(inflateEnd(&z) != Z_OK) { - err(" failed to deinit zlib stream"); + err(" failed to deinit zlib stream"); } + + // unpacked size + sz = z.total_out; } // if the blob has lzma-compressed data if(blob.has_lzma_data()) { // if the flag is raised issue a warning, a blob may only contain one data stream if(flag) - warn(" contains raw- and lzma-data at the same time"); + warn(" contains raw- and lzma-data at the same time"); // raise the flag - we have at least one datastream flag = true; // tell about the compressed data - debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); - debug(" uncompressed size: %u bytes", blob.raw_size()); + debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + debug(" uncompressed size: %u bytes", blob.raw_size()); // issue a warning, lzma compression is not yet supported - err(" lzma-decompression is not supported"); + err(" lzma-decompression is not supported"); } // check the flag is raised and we have at least one data-stream if(!flag) - err(" does not contain any known data stream"); + err(" does not contain any known data stream"); // lower the flag flag = false; + + // switch between different blob-types + if(blobheader.type() == "OSMHeader") { + // tell about the OSMHeader blob + info(" OSMHeader"); + + // parse the OSMHeader from the blob + osmheader.ParseFromArray(unpack_buffer, sz); + + // tell about the required features + for(int i = 0, l = osmheader.required_features_size(); i < l; i++) + debug(" required_feature: %s", osmheader.required_features(i).c_str()); + + // tell about the optional features + for(int i = 0, l = osmheader.optional_features_size(); i < l; i++) + debug(" required_feature: %s", osmheader.optional_features(i).c_str()); + + // tell about the writing program + debug(" writingprogram: %s", osmheader.writingprogram().c_str()); + + // tell about the source + debug(" source: %s", osmheader.source().c_str()); + } + + else if(blobheader.type() == "OSMData") { + // tell about the OSMData blob + info(" OSMData"); + } + + else { + // unknown blob type + warn(" unknown blob type: %s", blobheader.type().c_str()); + } } // close the file pointer From 072daaee0ad49f2c6c78b9cdab59897be85631e2 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:35:13 +0200 Subject: [PATCH 06/29] always color to stdout --- tools/osmpbf-outline.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 352c9f9..b47f004 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -5,8 +5,6 @@ #include #include -bool is_a_tty; - const int MAX_BLOB_HEADER_SIZE = 64 * 1024; const int MAX_BLOB_SIZE = 32 * 1024 * 1024; @@ -21,12 +19,9 @@ OSMPBF::HeaderBlock osmheader; * prints a formatted message to stderr, optionally color coded */ void msg(const char* format, int color, va_list args) { - if(is_a_tty) fprintf(stderr, "\x1b[0;%dm", color); - - vfprintf(stderr, format, args); - - if(is_a_tty) fprintf(stderr, "\x1b[0m\n"); - else fprintf(stderr, "\n"); + fprintf(stdout, "\x1b[0;%dm", color); + vfprintf(stdout, format, args); + fprintf(stdout, "\x1b[0m\n"); } /** @@ -74,9 +69,6 @@ void debug(const char* format, ...) { * application main method */ int main(int argc, char *argv[]) { - // check if stderr is a tty and set a global bool - is_a_tty = (bool)isatty(2); - // check for proper command line args if(argc != 2) err("usage: %s file.osm.pbf", argv[0]); From f5a5dde412c26dd551d3527c85064433629a71e3 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:35:40 +0200 Subject: [PATCH 07/29] stdout, i said! --- tools/osmpbf-outline.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index b47f004..e265c9c 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -16,7 +16,7 @@ OSMPBF::Blob blob; OSMPBF::HeaderBlock osmheader; /** - * prints a formatted message to stderr, optionally color coded + * prints a formatted message to stdout, optionally color coded */ void msg(const char* format, int color, va_list args) { fprintf(stdout, "\x1b[0;%dm", color); From 3727704d2bc877a9b4c61d9e61006b0eb128661d Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:44:54 +0200 Subject: [PATCH 08/29] even more documentation --- tools/osmpbf-outline.cpp | 48 +++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index e265c9c..2373b00 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -1,32 +1,50 @@ +// used for va_list in debug-print methods #include + +// file io lib #include + +// zlib compression is used inside the pbf blobs #include + +// netinet provides the network-byte-order conversion function #include + +// this is the header to libosmpbf for reading and writing the lowlevel blob storags #include + +// this is the header to libosmpbf for reading and writing the highlevel osm objects #include +// the maximum size of a blob-header in bytes const int MAX_BLOB_HEADER_SIZE = 64 * 1024; + +// the maximum size of a blob in bytes const int MAX_BLOB_SIZE = 32 * 1024 * 1024; +// buffer for reading a compressed blob from file char buffer[MAX_BLOB_SIZE]; + +// buffer for decompressing the blob char unpack_buffer[MAX_BLOB_SIZE]; +// pbf structur of a BlobHeader OSMPBF::BlobHeader blobheader; + +// pbf structur of a Blob OSMPBF::Blob blob; + +// pbf structur of a OSM HeaderBlock OSMPBF::HeaderBlock osmheader; -/** - * prints a formatted message to stdout, optionally color coded - */ +// prints a formatted message to stdout, optionally color coded void msg(const char* format, int color, va_list args) { fprintf(stdout, "\x1b[0;%dm", color); vfprintf(stdout, format, args); fprintf(stdout, "\x1b[0m\n"); } -/** - * prints a formatted message to stderr, color coded to red - */ +// prints a formatted message to stderr, color coded to red void err(const char* format, ...) { va_list args; va_start(args, format); @@ -35,9 +53,7 @@ void err(const char* format, ...) { exit(1); } -/** - * prints a formatted message to stderr, color coded to yellow - */ +// prints a formatted message to stderr, color coded to yellow void warn(const char* format, ...) { va_list args; va_start(args, format); @@ -45,9 +61,7 @@ void warn(const char* format, ...) { va_end(args); } -/** - * prints a formatted message to stderr, color coded to green - */ +// prints a formatted message to stderr, color coded to green void info(const char* format, ...) { va_list args; va_start(args, format); @@ -55,9 +69,7 @@ void info(const char* format, ...) { va_end(args); } -/** - * prints a formatted message to stderr, color coded to white - */ +// prints a formatted message to stderr, color coded to white void debug(const char* format, ...) { va_list args; va_start(args, format); @@ -65,9 +77,7 @@ void debug(const char* format, ...) { va_end(args); } -/** - * application main method - */ +// application main method int main(int argc, char *argv[]) { // check for proper command line args if(argc != 2) @@ -113,8 +123,6 @@ int main(int argc, char *argv[]) { // optional indexdata if(blobheader.has_indexdata()) debug(" indexdata = %u bytes", blobheader.indexdata().size()); - else - debug(" no indexdata"); // ensure the blob is smaller then MAX_BLOB_SIZE if(sz > MAX_BLOB_SIZE) From f1c905e60f326e2280967e8ce9de01e351e88b53 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:51:59 +0200 Subject: [PATCH 09/29] bbox reader --- tools/osmpbf-outline.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 2373b00..5d3d00a 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -22,6 +22,9 @@ const int MAX_BLOB_HEADER_SIZE = 64 * 1024; // the maximum size of a blob in bytes const int MAX_BLOB_SIZE = 32 * 1024 * 1024; +// nanodegree multiplier +static const long int NANO = 1000 * 1000 * 1000; + // buffer for reading a compressed blob from file char buffer[MAX_BLOB_SIZE]; @@ -239,6 +242,14 @@ int main(int argc, char *argv[]) { // parse the OSMHeader from the blob osmheader.ParseFromArray(unpack_buffer, sz); + // tell about the bbox + if(osmheader.has_bbox()) { + OSMPBF::HeaderBBox bbox = osmheader.bbox(); + debug(" bbox: %.7f,%.7f,%.7f,%.7f", + (double)bbox.left() / NANO, (double)bbox.bottom() / NANO, + (double)bbox.right() / NANO, (double)bbox.top() / NANO); + } + // tell about the required features for(int i = 0, l = osmheader.required_features_size(); i < l; i++) debug(" required_feature: %s", osmheader.required_features(i).c_str()); From 0fbe0f8cff1e624a7989bc61493dd65728bfd00c Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 21:54:19 +0200 Subject: [PATCH 10/29] make source & writingprogram optional --- tools/osmpbf-outline.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 5d3d00a..b2089ee 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -259,10 +259,12 @@ int main(int argc, char *argv[]) { debug(" required_feature: %s", osmheader.optional_features(i).c_str()); // tell about the writing program - debug(" writingprogram: %s", osmheader.writingprogram().c_str()); + if(osmheader.has_writingprogram()); + debug(" writingprogram: %s", osmheader.writingprogram().c_str()); // tell about the source - debug(" source: %s", osmheader.source().c_str()); + if(osmheader.has_source()) + debug(" source: %s", osmheader.source().c_str()); } else if(blobheader.type() == "OSMData") { From e2d430cbf947a373b0e94e26c43e59ac8f4c90f8 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 22:17:29 +0200 Subject: [PATCH 11/29] look into the block --- tools/osmpbf-outline.cpp | 86 ++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index b2089ee..3d303c7 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -31,14 +31,17 @@ char buffer[MAX_BLOB_SIZE]; // buffer for decompressing the blob char unpack_buffer[MAX_BLOB_SIZE]; -// pbf structur of a BlobHeader +// pbf struct of a BlobHeader OSMPBF::BlobHeader blobheader; -// pbf structur of a Blob +// pbf struct of a Blob OSMPBF::Blob blob; -// pbf structur of a OSM HeaderBlock -OSMPBF::HeaderBlock osmheader; +// pbf struct of an OSM HeaderBlock +OSMPBF::HeaderBlock headerblock; + +// pbf struct of an OSM PrimitiveBlock +OSMPBF::PrimitiveBlock primblock; // prints a formatted message to stdout, optionally color coded void msg(const char* format, int color, va_list args) { @@ -239,37 +242,87 @@ int main(int argc, char *argv[]) { // tell about the OSMHeader blob info(" OSMHeader"); - // parse the OSMHeader from the blob - osmheader.ParseFromArray(unpack_buffer, sz); + // parse the HeaderBlock from the blob + headerblock.ParseFromArray(unpack_buffer, sz); // tell about the bbox - if(osmheader.has_bbox()) { - OSMPBF::HeaderBBox bbox = osmheader.bbox(); + if(headerblock.has_bbox()) { + OSMPBF::HeaderBBox bbox = headerblock.bbox(); debug(" bbox: %.7f,%.7f,%.7f,%.7f", (double)bbox.left() / NANO, (double)bbox.bottom() / NANO, (double)bbox.right() / NANO, (double)bbox.top() / NANO); } // tell about the required features - for(int i = 0, l = osmheader.required_features_size(); i < l; i++) - debug(" required_feature: %s", osmheader.required_features(i).c_str()); + for(int i = 0, l = headerblock.required_features_size(); i < l; i++) + debug(" required_feature: %s", headerblock.required_features(i).c_str()); // tell about the optional features - for(int i = 0, l = osmheader.optional_features_size(); i < l; i++) - debug(" required_feature: %s", osmheader.optional_features(i).c_str()); + for(int i = 0, l = headerblock.optional_features_size(); i < l; i++) + debug(" required_feature: %s", headerblock.optional_features(i).c_str()); // tell about the writing program - if(osmheader.has_writingprogram()); - debug(" writingprogram: %s", osmheader.writingprogram().c_str()); + if(headerblock.has_writingprogram()); + debug(" writingprogram: %s", headerblock.writingprogram().c_str()); // tell about the source - if(osmheader.has_source()) - debug(" source: %s", osmheader.source().c_str()); + if(headerblock.has_source()) + debug(" source: %s", headerblock.source().c_str()); } else if(blobheader.type() == "OSMData") { // tell about the OSMData blob info(" OSMData"); + + // parse the PrimitiveBlock from the blob + primblock.ParseFromArray(unpack_buffer, sz); + + // tell about the stringtable + debug(" stringtable: %u items", primblock.stringtable().s_size()); + + // number of PrimitiveGroups + debug(" primitivegroups: %u groups", primblock.primitivegroup_size()); + + // iterate over all PrimitiveGroups + for(int i = 0, l = primblock.primitivegroup_size(); i < l; i++) { + // one PrimitiveGroup from the the Block + OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i); + + // tell about nodes + if(pg.nodes_size() > 0) { + // raise the flag - we have at least one item type + flag = true; + + debug(" nodes: %d", pg.nodes_size()); + } + + // tell about densenodes + if(pg.has_dense()) { + // raise the flag - we have at leastone item type + flag = true; + + debug(" dense nodes: %d", pg.dense().id_size()); + } + + // tell about ways + if(pg.ways_size() > 0) { + // raise the flag - we have at leastone item type + flag = true; + + debug(" ways: %d", pg.ways_size()); + } + + // tell aboutrelations + if(pg.relations_size() > 0) { + // raise the flag - we have at leastone item type + flag = true; + + debug(" relations: %d", pg.relations_size()); + } + + if(!flag) + warn(" contains no items"); + } } else { @@ -284,3 +337,4 @@ int main(int argc, char *argv[]) { // clean up the protobuf lib google::protobuf::ShutdownProtobufLibrary(); } + From a1f2b8133251e4a848449ca15b1ccc951e57fb17 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 23 May 2011 22:57:29 +0200 Subject: [PATCH 12/29] tell about meta-info --- tools/osmpbf-outline.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 3d303c7..f032d3b 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -294,6 +294,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" nodes: %d", pg.nodes_size()); + if(pg.nodes(0).has_info()) + debug(" with meta-info"); } // tell about densenodes @@ -302,6 +304,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" dense nodes: %d", pg.dense().id_size()); + if(pg.dense().has_denseinfo()) + debug(" with meta-info"); } // tell about ways @@ -310,6 +314,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" ways: %d", pg.ways_size()); + if(pg.ways(0).has_info()) + debug(" with meta-info"); } // tell aboutrelations @@ -318,6 +324,8 @@ int main(int argc, char *argv[]) { flag = true; debug(" relations: %d", pg.relations_size()); + if(pg.relations(0).has_info()) + debug(" with meta-info"); } if(!flag) From b6cf2110eca09d0a30ed80409c084b7fcdcab853 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 25 May 2011 21:28:43 +0200 Subject: [PATCH 13/29] show primblock metainfo --- tools/osmpbf-outline.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index f032d3b..af357d5 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -277,6 +277,12 @@ int main(int argc, char *argv[]) { // parse the PrimitiveBlock from the blob primblock.ParseFromArray(unpack_buffer, sz); + // tell about the block's meta info + debug(" granularity: %u", primblock.granularity()); + debug(" lat_offset: %u", primblock.lat_offset()); + debug(" lon_offset: %u", primblock.lon_offset()); + debug(" date_granularity: %u", primblock.date_granularity()); + // tell about the stringtable debug(" stringtable: %u items", primblock.stringtable().s_size()); From 1ddaca6a03afde8d16e4bf7ab3cd065f09c8d9b1 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 26 May 2011 16:02:38 +0200 Subject: [PATCH 14/29] extend definition with visible-flag for storing history information --- src/osmformat.proto | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/osmformat.proto b/src/osmformat.proto index ddf2c84..49c20a2 100644 --- a/src/osmformat.proto +++ b/src/osmformat.proto @@ -126,6 +126,16 @@ message Info { optional int64 changeset = 3; optional int32 uid = 4; optional uint32 user_sid = 5; // String IDs + + // the visible flag is used to store history information in .osm.pbf files. + // it indicates that the current object version has been created by a delete + // operation on the osm api. + + // when a writer sets this flag, it MUST add a required_features with value "HistoricalInformation" + // when a reader is able to work with the visible flag, it MUST check it's existence using has_visible and + // assume TRUE it it is not set. A reader that does not support the visible flag, it MUST dismiss files with + // the required_feature "HistoricalInformation". + optional bool visible = 6; } /** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ @@ -135,6 +145,14 @@ message DenseInfo { repeated sint64 changeset = 3 [packed = true]; // DELTA coded repeated sint32 uid = 4 [packed = true]; // DELTA coded repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded + + // this field is the dense pendent to Info.visible + + // when a writer sets this flag, it MUST add a required_features with value "HistoricalInformation" + // when a reader is able to work with the visible flag, it MUST check it's existence using has_visible and + // assume TRUE it it is not set. A reader that does not support the visible flag, it MUST dismiss files with + // the required_feature "HistoricalInformation". + repeated bool visible = 6 [packed = true]; } From dedab832868ffe0f2cf053bcc7ce32e92fb5b8fa Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Wed, 1 Jun 2011 14:06:44 +0200 Subject: [PATCH 15/29] fixed some descriptions --- tools/osmpbf-outline.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index af357d5..7a2130d 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -10,10 +10,10 @@ // netinet provides the network-byte-order conversion function #include -// this is the header to libosmpbf for reading and writing the lowlevel blob storags +// this is the header to libosmpbf for reading and writing the low-level blob storage #include -// this is the header to libosmpbf for reading and writing the highlevel osm objects +// this is the header to libosmpbf for reading and writing the high-level osm objects #include // the maximum size of a blob-header in bytes @@ -304,9 +304,9 @@ int main(int argc, char *argv[]) { debug(" with meta-info"); } - // tell about densenodes + // tell about dense nodes if(pg.has_dense()) { - // raise the flag - we have at leastone item type + // raise the flag - we have at least one item type flag = true; debug(" dense nodes: %d", pg.dense().id_size()); @@ -316,7 +316,7 @@ int main(int argc, char *argv[]) { // tell about ways if(pg.ways_size() > 0) { - // raise the flag - we have at leastone item type + // raise the flag - we have at least one item type flag = true; debug(" ways: %d", pg.ways_size()); @@ -324,9 +324,9 @@ int main(int argc, char *argv[]) { debug(" with meta-info"); } - // tell aboutrelations + // tell about relations if(pg.relations_size() > 0) { - // raise the flag - we have at leastone item type + // raise the flag - we have at least one item type flag = true; debug(" relations: %d", pg.relations_size()); From d383d89f9a09c7afdd6853fc3c2fdcd902120257 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Wed, 1 Jun 2011 14:21:06 +0200 Subject: [PATCH 16/29] Used two flag variables with proper names instead of one just called 'flag'. Add return before BlobHeader ouput. --- tools/osmpbf-outline.cpp | 55 ++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 7a2130d..2aec7f5 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -97,9 +97,6 @@ int main(int argc, char *argv[]) { // storage of size, used multiple times int32_t sz; - // boolean flag, used multiple times - bool flag = false; - // read the first 4 bytes of the file, this is the size of the blob-header if(fread(&sz, sizeof(sz), 1, fp) != 1) break; // end of file reached @@ -119,7 +116,7 @@ int main(int argc, char *argv[]) { blobheader.ParseFromArray(buffer, sz); // tell about the blob-header - info("BlobHeader (%d bytes)", sz); + info("\nBlobHeader (%d bytes)", sz); debug(" type = %s", blobheader.type().c_str()); // size of the following blob @@ -144,10 +141,13 @@ int main(int argc, char *argv[]) { // tell about the blob-header info("Blob (%d bytes)", sz); + // set when we find at least one data stream + bool found_data = false; + // if the blob has uncompressed data if(blob.has_raw()) { - // raise the flag - we have at least one datastream - flag = true; + // we have at least one datastream + found_data = true; // size of the blob-data sz = blob.raw().size(); @@ -165,12 +165,12 @@ int main(int argc, char *argv[]) { // if the blob has zlib-compressed data if(blob.has_zlib_data()) { - // if the flag is raised issue a warning, a blob may only contain one data stream - if(flag) - warn(" contains raw- and zlib-data at the same time"); + // issue a warning if there is more than one data steam, a blob may only contain one data stream + if(found_data) + warn(" contains several data streams"); - // raise the flag - we have at least one datastream - flag = true; + // we have at least one datastream + found_data = true; // the size of the compressesd data sz = blob.zlib_data().size(); @@ -215,12 +215,12 @@ int main(int argc, char *argv[]) { // if the blob has lzma-compressed data if(blob.has_lzma_data()) { - // if the flag is raised issue a warning, a blob may only contain one data stream - if(flag) - warn(" contains raw- and lzma-data at the same time"); + // issue a warning if there is more than one data steam, a blob may only contain one data stream + if(found_data) + warn(" contains several data streams"); - // raise the flag - we have at least one datastream - flag = true; + // we have at least one datastream + found_data = true; // tell about the compressed data debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); @@ -230,13 +230,10 @@ int main(int argc, char *argv[]) { err(" lzma-decompression is not supported"); } - // check the flag is raised and we have at least one data-stream - if(!flag) + // check we have at least one data-stream + if(!found_data) err(" does not contain any known data stream"); - // lower the flag - flag = false; - // switch between different blob-types if(blobheader.type() == "OSMHeader") { // tell about the OSMHeader blob @@ -294,10 +291,11 @@ int main(int argc, char *argv[]) { // one PrimitiveGroup from the the Block OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i); + bool found_items=false; + // tell about nodes if(pg.nodes_size() > 0) { - // raise the flag - we have at least one item type - flag = true; + found_items = true; debug(" nodes: %d", pg.nodes_size()); if(pg.nodes(0).has_info()) @@ -306,8 +304,7 @@ int main(int argc, char *argv[]) { // tell about dense nodes if(pg.has_dense()) { - // raise the flag - we have at least one item type - flag = true; + found_items = true; debug(" dense nodes: %d", pg.dense().id_size()); if(pg.dense().has_denseinfo()) @@ -316,8 +313,7 @@ int main(int argc, char *argv[]) { // tell about ways if(pg.ways_size() > 0) { - // raise the flag - we have at least one item type - flag = true; + found_items = true; debug(" ways: %d", pg.ways_size()); if(pg.ways(0).has_info()) @@ -326,15 +322,14 @@ int main(int argc, char *argv[]) { // tell about relations if(pg.relations_size() > 0) { - // raise the flag - we have at least one item type - flag = true; + found_items = true; debug(" relations: %d", pg.relations_size()); if(pg.relations(0).has_info()) debug(" with meta-info"); } - if(!flag) + if(!found_items) warn(" contains no items"); } } From 6554a40cdf23ee059caadb30c0ab95648f897873 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Wed, 1 Jun 2011 14:40:24 +0200 Subject: [PATCH 17/29] Add osmpbf.h The idea is that this file can be included and will include all needed other files and contain some definitions. Not used yet, though. --- .gitignore | 1 + src/osmpbf.h | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 src/osmpbf.h diff --git a/.gitignore b/.gitignore index c016218..f3aa25e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ src/*.pb.h src/*.pb.o src/libosmpbf.a +*.swp diff --git a/src/osmpbf.h b/src/osmpbf.h new file mode 100644 index 0000000..6bff1a8 --- /dev/null +++ b/src/osmpbf.h @@ -0,0 +1,20 @@ +#ifndef OSMPBF_H +#define OSMPBF_H + +// this describes the low-level blob storage +#include + +// this describes the high-level OSM objects +#include + +namespace osmpbf { + + // the maximum size of a blob header in bytes + const int max_blob_header_size = 64 * 1024; + + // the maximum size of a blob in bytes + const int max_uncompressed_blob_size = 32 * 1024 * 1024; + +} + +#endif // OSMPBF_H From b3586a7b1909ff27607ec682177e78bb8c50e5f6 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Wed, 1 Jun 2011 16:36:37 +0200 Subject: [PATCH 18/29] Improved description for new visible flag --- src/osmformat.proto | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/osmformat.proto b/src/osmformat.proto index 49c20a2..48a3575 100644 --- a/src/osmformat.proto +++ b/src/osmformat.proto @@ -127,14 +127,14 @@ message Info { optional int32 uid = 4; optional uint32 user_sid = 5; // String IDs - // the visible flag is used to store history information in .osm.pbf files. - // it indicates that the current object version has been created by a delete - // operation on the osm api. - - // when a writer sets this flag, it MUST add a required_features with value "HistoricalInformation" - // when a reader is able to work with the visible flag, it MUST check it's existence using has_visible and - // assume TRUE it it is not set. A reader that does not support the visible flag, it MUST dismiss files with - // the required_feature "HistoricalInformation". + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. optional bool visible = 6; } @@ -146,12 +146,14 @@ message DenseInfo { repeated sint32 uid = 4 [packed = true]; // DELTA coded repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded - // this field is the dense pendent to Info.visible - - // when a writer sets this flag, it MUST add a required_features with value "HistoricalInformation" - // when a reader is able to work with the visible flag, it MUST check it's existence using has_visible and - // assume TRUE it it is not set. A reader that does not support the visible flag, it MUST dismiss files with - // the required_feature "HistoricalInformation". + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. repeated bool visible = 6 [packed = true]; } From 282bb229f502985f68e1c7f3552b264db4193a7d Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 3 Jun 2011 12:05:44 +0200 Subject: [PATCH 19/29] New osmpbf.h and new debian package. * New osmpbf.h makes including easier and defines some often used constants * README now explains how to use the lib * New version of debian package includes * osmpbf.h * osmpbf-outline tool * README --- README | 7 +++++++ debian/changelog | 8 ++++++++ debian/docs | 1 + debian/rules | 3 +++ src/Makefile | 2 +- src/osmpbf.h | 15 +++++++++++---- tools/Makefile | 7 +++---- 7 files changed, 34 insertions(+), 9 deletions(-) diff --git a/README b/README index f09e399..d579913 100644 --- a/README +++ b/README @@ -4,6 +4,13 @@ OSM PBF See http://wiki.openstreetmap.org/wiki/PBF_Format . +To include in your program use: + +#include + +and link with: + -lpthread -lz -lprotobuf-lite -losmpbf + To build the Debian/Ubuntu package call: debuild -I -us -uc diff --git a/debian/changelog b/debian/changelog index 7599db6..c4bf50b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +libosmpbf-dev (1.1.1j1) maverick; urgency=low + + * Add support for OSM history file + * Add some often used constants to OSMPBF namespace + * Add osmpbf-outline testing tool + + -- Jochen Topf Fri, 3 Jun 2011 11:36:17 +0100 + libosmpbf-dev (0.1) maverick; urgency=low * Initial debian package diff --git a/debian/docs b/debian/docs index fe686d9..fb7cedf 100644 --- a/debian/docs +++ b/debian/docs @@ -1,2 +1,3 @@ src/fileformat.proto src/osmformat.proto +README diff --git a/debian/rules b/debian/rules index 72cb333..093785a 100755 --- a/debian/rules +++ b/debian/rules @@ -24,6 +24,7 @@ build-stamp: configure-stamp # Add here commands to compile the package. $(MAKE) -C src + $(MAKE) -C tools touch $@ @@ -34,6 +35,7 @@ clean: # Add here commands to clean up after the build process. $(MAKE) -C src clean || /bin/true + $(MAKE) -C tools clean || /bin/true dh_clean @@ -45,6 +47,7 @@ install: build # Add here commands to install the package into debian/libosmpbf-dev. $(MAKE) -C src DESTDIR=$(CURDIR)/debian/libosmpbf-dev install + $(MAKE) -C tools DESTDIR=$(CURDIR)/debian/libosmpbf-dev install # Build architecture-independent files here. diff --git a/src/Makefile b/src/Makefile index efd938a..37382b2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,7 +18,7 @@ install: install -m 755 -g root -o root -d $(DESTDIR)/usr/lib install -m 644 -g root -o root libosmpbf.a $(DESTDIR)/usr/lib install -m 755 -g root -o root -d $(DESTDIR)/usr/include/osmpbf - install -m 644 -g root -o root fileformat.pb.h osmformat.pb.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root osmpbf.h fileformat.pb.h osmformat.pb.h $(DESTDIR)/usr/include/osmpbf clean: rm -f *.pb.h *.pb.cc *.pb.o libosmpbf.a diff --git a/src/osmpbf.h b/src/osmpbf.h index 6bff1a8..ce6a7b1 100644 --- a/src/osmpbf.h +++ b/src/osmpbf.h @@ -7,13 +7,20 @@ // this describes the high-level OSM objects #include -namespace osmpbf { +namespace OSMPBF { + + // library version + const char *version = "1.1.1j1"; // the maximum size of a blob header in bytes - const int max_blob_header_size = 64 * 1024; + const int max_blob_header_size = 64 * 1024; // 64 kB - // the maximum size of a blob in bytes - const int max_uncompressed_blob_size = 32 * 1024 * 1024; + // the maximum size of an uncompressed blob in bytes + const int max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB + + // resolution for longitude/latitude used for conversion + // between representation as double and as int + const int lonlat_resolution = 1000 * 1000 * 1000; } diff --git a/tools/Makefile b/tools/Makefile index 771ef9c..9598dcf 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,17 +1,16 @@ CXX = g++ CXXFLAGS = -O3 -LDFLAGS = -lpthread -LIB_PROTOBUF = -lz -lprotobuf-lite -losmpbf +LDFLAGS = -lpthread -lz -lprotobuf-lite -losmpbf all: osmpbf-outline osmpbf-outline: osmpbf-outline.cpp - $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) $(LIB_PROTOBUF) + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) install: install -m 755 -g root -o root -d $(DESTDIR)/usr/bin - install -m 644 -g root -o root osmpbf-outline $(DESTDIR)/usr/bin + install -m 644 -g root -o root -s osmpbf-outline $(DESTDIR)/usr/bin clean: rm -f osmpbf-outline From 187e1f900b699879239ccf23e5e7ef57b4b29ff6 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sun, 5 Jun 2011 11:51:11 +0200 Subject: [PATCH 20/29] add magic file --- magic | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 magic diff --git a/magic b/magic new file mode 100644 index 0000000..9bdff0d --- /dev/null +++ b/magic @@ -0,0 +1,3 @@ +# Magic data for file(1) command. +# Format is described in magic(5). +6 string OSMHeader OpenStreetMap PBF From 4143aede64f129f412ada08bdd45f8934e1584c4 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 5 Jun 2011 18:49:58 +0200 Subject: [PATCH 21/29] use osmpbf.h --- tools/osmpbf-outline.cpp | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 2aec7f5..4094e73 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -10,26 +10,14 @@ // netinet provides the network-byte-order conversion function #include -// this is the header to libosmpbf for reading and writing the low-level blob storage -#include - -// this is the header to libosmpbf for reading and writing the high-level osm objects -#include - -// the maximum size of a blob-header in bytes -const int MAX_BLOB_HEADER_SIZE = 64 * 1024; - -// the maximum size of a blob in bytes -const int MAX_BLOB_SIZE = 32 * 1024 * 1024; - -// nanodegree multiplier -static const long int NANO = 1000 * 1000 * 1000; +// this is the header to pbf format +#include // buffer for reading a compressed blob from file -char buffer[MAX_BLOB_SIZE]; +char buffer[OSMPBF::max_uncompressed_blob_size]; // buffer for decompressing the blob -char unpack_buffer[MAX_BLOB_SIZE]; +char unpack_buffer[OSMPBF::max_uncompressed_blob_size]; // pbf struct of a BlobHeader OSMPBF::BlobHeader blobheader; @@ -105,8 +93,8 @@ int main(int argc, char *argv[]) { sz = ntohl(sz); // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE - if(sz > MAX_BLOB_HEADER_SIZE) - err("blob-header-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_HEADER_SIZE); + if(sz > OSMPBF::max_blob_header_size) + err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size); // read the blob-header from the file if(fread(buffer, sz, 1, fp) != 1) @@ -128,8 +116,8 @@ int main(int argc, char *argv[]) { debug(" indexdata = %u bytes", blobheader.indexdata().size()); // ensure the blob is smaller then MAX_BLOB_SIZE - if(sz > MAX_BLOB_SIZE) - err("blob-size is bigger then allowed (%u > %u)", sz, MAX_BLOB_SIZE); + if(sz > OSMPBF::max_uncompressed_blob_size) + err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size); // read the blob from the file if(fread(buffer, sz, 1, fp) != 1) @@ -246,8 +234,10 @@ int main(int argc, char *argv[]) { if(headerblock.has_bbox()) { OSMPBF::HeaderBBox bbox = headerblock.bbox(); debug(" bbox: %.7f,%.7f,%.7f,%.7f", - (double)bbox.left() / NANO, (double)bbox.bottom() / NANO, - (double)bbox.right() / NANO, (double)bbox.top() / NANO); + (double)bbox.left() / OSMPBF::lonlat_resolution, + (double)bbox.bottom() / OSMPBF::lonlat_resolution, + (double)bbox.right() / OSMPBF::lonlat_resolution, + (double)bbox.top() / OSMPBF::lonlat_resolution); } // tell about the required features From 7aaa1a5e509bfb3cc46418d48c74e1597b77922e Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sun, 5 Jun 2011 19:22:24 +0200 Subject: [PATCH 22/29] Fixed build of osmpbf-outline tool --- .gitignore | 1 + {src => include/osmpbf}/osmpbf.h | 0 src/Makefile | 9 ++++++--- tools/Makefile | 4 ++-- tools/osmpbf-outline.cpp | 6 +----- 5 files changed, 10 insertions(+), 10 deletions(-) rename {src => include/osmpbf}/osmpbf.h (100%) diff --git a/.gitignore b/.gitignore index f3aa25e..c7ed701 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +include/osmpbf/*.pb.h src/*.pb.h src/*.pb.o src/libosmpbf.a diff --git a/src/osmpbf.h b/include/osmpbf/osmpbf.h similarity index 100% rename from src/osmpbf.h rename to include/osmpbf/osmpbf.h diff --git a/src/Makefile b/src/Makefile index 37382b2..16c3e97 100644 --- a/src/Makefile +++ b/src/Makefile @@ -3,7 +3,7 @@ CXX = g++ CXXFLAGS = -O3 AR = ar -all: libosmpbf.a fileformat.pb.h osmformat.pb.h +all: libosmpbf.a ../include/osmpbf/fileformat.pb.h ../include/osmpbf/osmformat.pb.h libosmpbf.a: fileformat.pb.o osmformat.pb.o $(AR) -cr $@ fileformat.pb.o osmformat.pb.o @@ -11,14 +11,17 @@ libosmpbf.a: fileformat.pb.o osmformat.pb.o %.pb.o: %.pb.cc $(CXX) $(CXXFLAGS) -c -o $@ $< -%.pb.cc %.pb.h: %.proto +%.pb.cc ../include/osmpbf/%.pb.h: %.proto protoc --proto_path=. --cpp_out=. $< + cp *.pb.h ../include/osmpbf/ install: install -m 755 -g root -o root -d $(DESTDIR)/usr/lib install -m 644 -g root -o root libosmpbf.a $(DESTDIR)/usr/lib install -m 755 -g root -o root -d $(DESTDIR)/usr/include/osmpbf - install -m 644 -g root -o root osmpbf.h fileformat.pb.h osmformat.pb.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root ../include/osmpbf/osmpbf.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root ../include/osmpbf/fileformat.pb.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root ../include/osmpbf/osmformat.pb.h $(DESTDIR)/usr/include/osmpbf clean: rm -f *.pb.h *.pb.cc *.pb.o libosmpbf.a diff --git a/tools/Makefile b/tools/Makefile index 9598dcf..25be30c 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,7 +1,7 @@ CXX = g++ -CXXFLAGS = -O3 -LDFLAGS = -lpthread -lz -lprotobuf-lite -losmpbf +CXXFLAGS = -O3 -I../include +LDFLAGS = -L../src -lpthread -lz -lprotobuf-lite -losmpbf all: osmpbf-outline diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 2aec7f5..46deaf4 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -10,11 +10,7 @@ // netinet provides the network-byte-order conversion function #include -// this is the header to libosmpbf for reading and writing the low-level blob storage -#include - -// this is the header to libosmpbf for reading and writing the high-level osm objects -#include +#include // the maximum size of a blob-header in bytes const int MAX_BLOB_HEADER_SIZE = 64 * 1024; From 7d52ed25beb2c368cfc2f0977f06a5c4e70ece1e Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 5 Jun 2011 20:35:15 +0200 Subject: [PATCH 23/29] disable colorization on !tty devices, add option to force colorization --- tools/osmpbf-outline.cpp | 44 +++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 4094e73..4fe50dc 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -4,6 +4,9 @@ // file io lib #include +// getopt is used to check for the --color-flag +#include + // zlib compression is used inside the pbf blobs #include @@ -13,6 +16,9 @@ // this is the header to pbf format #include +// should the output use color? +bool usecolor = false; + // buffer for reading a compressed blob from file char buffer[OSMPBF::max_uncompressed_blob_size]; @@ -33,9 +39,15 @@ OSMPBF::PrimitiveBlock primblock; // prints a formatted message to stdout, optionally color coded void msg(const char* format, int color, va_list args) { - fprintf(stdout, "\x1b[0;%dm", color); + if(usecolor) { + fprintf(stdout, "\x1b[0;%dm", color); + } vfprintf(stdout, format, args); - fprintf(stdout, "\x1b[0m\n"); + if(usecolor) { + fprintf(stdout, "\x1b[0m\n"); + } else { + fprintf(stdout, "\n"); + } } // prints a formatted message to stderr, color coded to red @@ -73,12 +85,34 @@ void debug(const char* format, ...) { // application main method int main(int argc, char *argv[]) { + // check if the output is a tty so we can use colors + usecolor = isatty(1); + + static struct option long_options[] = { + {"color", no_argument, 0, 'c'}, + }; + + while (1) { + int c = getopt_long(argc, argv, "c", long_options, 0); + + if (c == -1) + break; + + switch (c) { + case 'c': + usecolor = true; + break; + default: + exit(1); + } + } + // check for proper command line args - if(argc != 2) - err("usage: %s file.osm.pbf", argv[0]); + if(optind != argc-1) + err("usage: %s [--color] file.osm.pbf", argv[0]); // open specified file - FILE *fp = fopen(argv[1], "r"); + FILE *fp = fopen(argv[optind], "r"); // read while the file has not reached its end while(!feof(fp)) { From 1a6d6a35902b56c9b2f8e6fec34613a9b1d09912 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 5 Jun 2011 23:49:29 +0200 Subject: [PATCH 24/29] add manpage --- tools/osmpbf-outline.1 | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tools/osmpbf-outline.1 diff --git a/tools/osmpbf-outline.1 b/tools/osmpbf-outline.1 new file mode 100644 index 0000000..bd2401e --- /dev/null +++ b/tools/osmpbf-outline.1 @@ -0,0 +1,49 @@ +.TH osmpbf-outline 1 LOCAL + +.SH NAME + +osmpbf-outline - outline the content of an .osm.pbf or .osh.pbf file + +.SH SYNOPSIS + +.B osmpbf-outline +[ +.B --color +] +.I filename + +.SH DESCRIPTION + +.I osmpbf-outline +generates an colored outline an .osm.pbf or .osh.pbf file, revealing its inner structure. +It does not decode all information nor does it check for of possible misstakes, but it will +warn on some bad misstakes one can make when generating .osm.pbf files. + +This tool can be used by authors of .osm.pbf readers and writers and is useful to understand +the inner blob/block structure of .osm.pbf files. + +Its sourcecode is heavily documented and can be used as a good start on how to read- and write +.osm.pbf files with C++ + +.SH OPTIONS +.TP 5 + +.B -c --color +usually colorization is disabled when writing to a file descriptor that is no tty (eg. when +the output is piped to a file or to a pager like to more or less). Some pagers can handle ANSI +color code ( +.I more +can, less can when invoked as +.I less -R +). To enforce colorization when working with such pagers, specify the +.B --color +flag. + +.SH EXAMPLES + +osmpbf-outline germany.osm.pbf + +.SH AUTHOR + Peter Koerner + Jochen Topf + From d791c3ec1b9627e658e3c115a23aa062635613d1 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Mon, 6 Jun 2011 10:07:51 +0200 Subject: [PATCH 25/29] fix manpage --- tools/osmpbf-outline.1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/osmpbf-outline.1 b/tools/osmpbf-outline.1 index bd2401e..87ab4b3 100644 --- a/tools/osmpbf-outline.1 +++ b/tools/osmpbf-outline.1 @@ -16,14 +16,14 @@ osmpbf-outline - outline the content of an .osm.pbf or .osh.pbf file .I osmpbf-outline generates an colored outline an .osm.pbf or .osh.pbf file, revealing its inner structure. -It does not decode all information nor does it check for of possible misstakes, but it will -warn on some bad misstakes one can make when generating .osm.pbf files. +It does not decode all information nor does it check for of possible mistakes, but it will +warn on some bad mistakes one can make when generating .osm.pbf files. This tool can be used by authors of .osm.pbf readers and writers and is useful to understand the inner blob/block structure of .osm.pbf files. -Its sourcecode is heavily documented and can be used as a good start on how to read- and write -.osm.pbf files with C++ +Its source code is heavily documented and can be used as a good start on how to read- and +write .osm.pbf files with C++. .SH OPTIONS .TP 5 From 4d8a862cf95b361f36e0daf50b68228eee82a966 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Mon, 6 Jun 2011 10:31:41 +0200 Subject: [PATCH 26/29] Add manpage to debian package --- debian/changelog | 6 ++++++ debian/libosmpbf-dev.manpages | 1 + 2 files changed, 7 insertions(+) create mode 100644 debian/libosmpbf-dev.manpages diff --git a/debian/changelog b/debian/changelog index c4bf50b..42274e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +libosmpbf-dev (1.1.1j2) maverick; urgency=low + + * Add manpage for osmpbf-outline tool + + -- Jochen Topf Fri, 6 Jun 2011 10:30:12 +0100 + libosmpbf-dev (1.1.1j1) maverick; urgency=low * Add support for OSM history file diff --git a/debian/libosmpbf-dev.manpages b/debian/libosmpbf-dev.manpages new file mode 100644 index 0000000..e82c242 --- /dev/null +++ b/debian/libosmpbf-dev.manpages @@ -0,0 +1 @@ +tools/osmpbf-outline.1 From 3479ac3f8664679ac40a345904dff4554d93469d Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Mon, 6 Jun 2011 12:06:35 +0200 Subject: [PATCH 27/29] Make osmpbf-outline more robust Add -g header for debugging. Remove -O3 header because it led to a segfault on a corrupted PBF file in the fread() call. Strange! Check result of ParseFromArray functions. --- tools/Makefile | 2 +- tools/osmpbf-outline.cpp | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/Makefile b/tools/Makefile index 25be30c..b9fc26b 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,6 +1,6 @@ CXX = g++ -CXXFLAGS = -O3 -I../include +CXXFLAGS = -g -I../include LDFLAGS = -L../src -lpthread -lz -lprotobuf-lite -losmpbf all: osmpbf-outline diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index 4fe50dc..edd6e2f 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -135,7 +135,8 @@ int main(int argc, char *argv[]) { err("unable to read blob-header from file"); // parse the blob-header from the read-buffer - blobheader.ParseFromArray(buffer, sz); + if(!blobheader.ParseFromArray(buffer, sz)) + err("unable to parse blob header"); // tell about the blob-header info("\nBlobHeader (%d bytes)", sz); @@ -158,7 +159,8 @@ int main(int argc, char *argv[]) { err("unable to read blob from file"); // parse the blob from the read-buffer - blob.ParseFromArray(buffer, sz); + if(!blob.ParseFromArray(buffer, sz)) + err("unable to parse blob"); // tell about the blob-header info("Blob (%d bytes)", sz); @@ -262,7 +264,8 @@ int main(int argc, char *argv[]) { info(" OSMHeader"); // parse the HeaderBlock from the blob - headerblock.ParseFromArray(unpack_buffer, sz); + if(!headerblock.ParseFromArray(unpack_buffer, sz)) + err("unable to parse header block"); // tell about the bbox if(headerblock.has_bbox()) { @@ -296,7 +299,8 @@ int main(int argc, char *argv[]) { info(" OSMData"); // parse the PrimitiveBlock from the blob - primblock.ParseFromArray(unpack_buffer, sz); + if(!primblock.ParseFromArray(unpack_buffer, sz)) + err("unable to parse primitive block"); // tell about the block's meta info debug(" granularity: %u", primblock.granularity()); From 77bb51d3973d614516a1281865acbb69325195ed Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Sun, 12 Jun 2011 10:52:22 +0200 Subject: [PATCH 28/29] terminate longopt list to prevent crashes on FreeBSD --- tools/osmpbf-outline.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp index edd6e2f..d72bd55 100644 --- a/tools/osmpbf-outline.cpp +++ b/tools/osmpbf-outline.cpp @@ -90,6 +90,7 @@ int main(int argc, char *argv[]) { static struct option long_options[] = { {"color", no_argument, 0, 'c'}, + {0,0,0,0} }; while (1) { From e102a594b046e80a4d58cdf0498fdbc080e953a9 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Thu, 4 Aug 2011 17:03:46 +0200 Subject: [PATCH 29/29] Compilation instructions --- README | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/README b/README index d579913..5006cda 100644 --- a/README +++ b/README @@ -4,6 +4,25 @@ OSM PBF See http://wiki.openstreetmap.org/wiki/PBF_Format . +There is a Java and a C version of the PBF library code here. + +C Version +========= + +To compile: + cd src + make + +To install: + cd src + make install + +To build the Debian/Ubuntu package call: + debuild -I -us -uc + +To install the Debian/Ubuntu package call: + sudo dpkg --install ../libosmpbf-dev*.deb + To include in your program use: #include @@ -11,6 +30,3 @@ To include in your program use: and link with: -lpthread -lz -lprotobuf-lite -losmpbf -To build the Debian/Ubuntu package call: - debuild -I -us -uc -