diff --git a/.gitignore b/.gitignore index c016218..c7ed701 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +include/osmpbf/*.pb.h src/*.pb.h src/*.pb.o src/libosmpbf.a +*.swp diff --git a/README b/README index f09e399..5006cda 100644 --- a/README +++ b/README @@ -4,6 +4,29 @@ OSM PBF See http://wiki.openstreetmap.org/wiki/PBF_Format . -To build the Debian/Ubuntu package call: - debuild -I -us -uc +There is a Java and a C version of the PBF library code here. + +C Version +========= + +To compile: + cd src + make + +To install: + cd src + make install + +To build the Debian/Ubuntu package call: + debuild -I -us -uc + +To install the Debian/Ubuntu package call: + sudo dpkg --install ../libosmpbf-dev*.deb + +To include in your program use: + +#include + +and link with: + -lpthread -lz -lprotobuf-lite -losmpbf diff --git a/debian/changelog b/debian/changelog index 7599db6..42274e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +libosmpbf-dev (1.1.1j2) maverick; urgency=low + + * Add manpage for osmpbf-outline tool + + -- Jochen Topf Fri, 6 Jun 2011 10:30:12 +0100 + +libosmpbf-dev (1.1.1j1) maverick; urgency=low + + * Add support for OSM history file + * Add some often used constants to OSMPBF namespace + * Add osmpbf-outline testing tool + + -- Jochen Topf Fri, 3 Jun 2011 11:36:17 +0100 + libosmpbf-dev (0.1) maverick; urgency=low * Initial debian package diff --git a/debian/docs b/debian/docs index fe686d9..fb7cedf 100644 --- a/debian/docs +++ b/debian/docs @@ -1,2 +1,3 @@ src/fileformat.proto src/osmformat.proto +README diff --git a/debian/libosmpbf-dev.manpages b/debian/libosmpbf-dev.manpages new file mode 100644 index 0000000..e82c242 --- /dev/null +++ b/debian/libosmpbf-dev.manpages @@ -0,0 +1 @@ +tools/osmpbf-outline.1 diff --git a/debian/rules b/debian/rules index 72cb333..093785a 100755 --- a/debian/rules +++ b/debian/rules @@ -24,6 +24,7 @@ build-stamp: configure-stamp # Add here commands to compile the package. $(MAKE) -C src + $(MAKE) -C tools touch $@ @@ -34,6 +35,7 @@ clean: # Add here commands to clean up after the build process. $(MAKE) -C src clean || /bin/true + $(MAKE) -C tools clean || /bin/true dh_clean @@ -45,6 +47,7 @@ install: build # Add here commands to install the package into debian/libosmpbf-dev. $(MAKE) -C src DESTDIR=$(CURDIR)/debian/libosmpbf-dev install + $(MAKE) -C tools DESTDIR=$(CURDIR)/debian/libosmpbf-dev install # Build architecture-independent files here. diff --git a/include/osmpbf/osmpbf.h b/include/osmpbf/osmpbf.h new file mode 100644 index 0000000..ce6a7b1 --- /dev/null +++ b/include/osmpbf/osmpbf.h @@ -0,0 +1,27 @@ +#ifndef OSMPBF_H +#define OSMPBF_H + +// this describes the low-level blob storage +#include + +// this describes the high-level OSM objects +#include + +namespace OSMPBF { + + // library version + const char *version = "1.1.1j1"; + + // the maximum size of a blob header in bytes + const int max_blob_header_size = 64 * 1024; // 64 kB + + // the maximum size of an uncompressed blob in bytes + const int max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB + + // resolution for longitude/latitude used for conversion + // between representation as double and as int + const int lonlat_resolution = 1000 * 1000 * 1000; + +} + +#endif // OSMPBF_H diff --git a/magic b/magic new file mode 100644 index 0000000..9bdff0d --- /dev/null +++ b/magic @@ -0,0 +1,3 @@ +# Magic data for file(1) command. +# Format is described in magic(5). +6 string OSMHeader OpenStreetMap PBF diff --git a/src/Makefile b/src/Makefile index efd938a..16c3e97 100644 --- a/src/Makefile +++ b/src/Makefile @@ -3,7 +3,7 @@ CXX = g++ CXXFLAGS = -O3 AR = ar -all: libosmpbf.a fileformat.pb.h osmformat.pb.h +all: libosmpbf.a ../include/osmpbf/fileformat.pb.h ../include/osmpbf/osmformat.pb.h libosmpbf.a: fileformat.pb.o osmformat.pb.o $(AR) -cr $@ fileformat.pb.o osmformat.pb.o @@ -11,14 +11,17 @@ libosmpbf.a: fileformat.pb.o osmformat.pb.o %.pb.o: %.pb.cc $(CXX) $(CXXFLAGS) -c -o $@ $< -%.pb.cc %.pb.h: %.proto +%.pb.cc ../include/osmpbf/%.pb.h: %.proto protoc --proto_path=. --cpp_out=. $< + cp *.pb.h ../include/osmpbf/ install: install -m 755 -g root -o root -d $(DESTDIR)/usr/lib install -m 644 -g root -o root libosmpbf.a $(DESTDIR)/usr/lib install -m 755 -g root -o root -d $(DESTDIR)/usr/include/osmpbf - install -m 644 -g root -o root fileformat.pb.h osmformat.pb.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root ../include/osmpbf/osmpbf.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root ../include/osmpbf/fileformat.pb.h $(DESTDIR)/usr/include/osmpbf + install -m 644 -g root -o root ../include/osmpbf/osmformat.pb.h $(DESTDIR)/usr/include/osmpbf clean: rm -f *.pb.h *.pb.cc *.pb.o libosmpbf.a diff --git a/src/osmformat.proto b/src/osmformat.proto index ddf2c84..48a3575 100644 --- a/src/osmformat.proto +++ b/src/osmformat.proto @@ -126,6 +126,16 @@ message Info { optional int64 changeset = 3; optional int32 uid = 4; optional uint32 user_sid = 5; // String IDs + + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. + optional bool visible = 6; } /** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ @@ -135,6 +145,16 @@ message DenseInfo { repeated sint64 changeset = 3 [packed = true]; // DELTA coded repeated sint32 uid = 4 [packed = true]; // DELTA coded repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded + + // The visible flag is used to store history information. It indicates that + // the current object version has been created by a delete operation on the + // OSM API. + // When a writer sets this flag, it MUST add a required_features tag with + // value "HistoricalInformation" to the HeaderBlock. + // If this flag is not available for some object it MUST be assumed to be + // true if the file has the required_features tag "HistoricalInformation" + // set. + repeated bool visible = 6 [packed = true]; } diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000..3235067 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1 @@ +osmpbf-outline diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..b9fc26b --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,16 @@ + +CXX = g++ +CXXFLAGS = -g -I../include +LDFLAGS = -L../src -lpthread -lz -lprotobuf-lite -losmpbf + +all: osmpbf-outline + +osmpbf-outline: osmpbf-outline.cpp + $(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS) + +install: + install -m 755 -g root -o root -d $(DESTDIR)/usr/bin + install -m 644 -g root -o root -s osmpbf-outline $(DESTDIR)/usr/bin + +clean: + rm -f osmpbf-outline diff --git a/tools/osmpbf-outline.1 b/tools/osmpbf-outline.1 new file mode 100644 index 0000000..87ab4b3 --- /dev/null +++ b/tools/osmpbf-outline.1 @@ -0,0 +1,49 @@ +.TH osmpbf-outline 1 LOCAL + +.SH NAME + +osmpbf-outline - outline the content of an .osm.pbf or .osh.pbf file + +.SH SYNOPSIS + +.B osmpbf-outline +[ +.B --color +] +.I filename + +.SH DESCRIPTION + +.I osmpbf-outline +generates an colored outline an .osm.pbf or .osh.pbf file, revealing its inner structure. +It does not decode all information nor does it check for of possible mistakes, but it will +warn on some bad mistakes one can make when generating .osm.pbf files. + +This tool can be used by authors of .osm.pbf readers and writers and is useful to understand +the inner blob/block structure of .osm.pbf files. + +Its source code is heavily documented and can be used as a good start on how to read- and +write .osm.pbf files with C++. + +.SH OPTIONS +.TP 5 + +.B -c --color +usually colorization is disabled when writing to a file descriptor that is no tty (eg. when +the output is piped to a file or to a pager like to more or less). Some pagers can handle ANSI +color code ( +.I more +can, less can when invoked as +.I less -R +). To enforce colorization when working with such pagers, specify the +.B --color +flag. + +.SH EXAMPLES + +osmpbf-outline germany.osm.pbf + +.SH AUTHOR + Peter Koerner + Jochen Topf + diff --git a/tools/osmpbf-outline.cpp b/tools/osmpbf-outline.cpp new file mode 100644 index 0000000..d72bd55 --- /dev/null +++ b/tools/osmpbf-outline.cpp @@ -0,0 +1,378 @@ +// used for va_list in debug-print methods +#include + +// file io lib +#include + +// getopt is used to check for the --color-flag +#include + +// zlib compression is used inside the pbf blobs +#include + +// netinet provides the network-byte-order conversion function +#include + +// this is the header to pbf format +#include + +// should the output use color? +bool usecolor = false; + +// buffer for reading a compressed blob from file +char buffer[OSMPBF::max_uncompressed_blob_size]; + +// buffer for decompressing the blob +char unpack_buffer[OSMPBF::max_uncompressed_blob_size]; + +// pbf struct of a BlobHeader +OSMPBF::BlobHeader blobheader; + +// pbf struct of a Blob +OSMPBF::Blob blob; + +// pbf struct of an OSM HeaderBlock +OSMPBF::HeaderBlock headerblock; + +// pbf struct of an OSM PrimitiveBlock +OSMPBF::PrimitiveBlock primblock; + +// prints a formatted message to stdout, optionally color coded +void msg(const char* format, int color, va_list args) { + if(usecolor) { + fprintf(stdout, "\x1b[0;%dm", color); + } + vfprintf(stdout, format, args); + if(usecolor) { + fprintf(stdout, "\x1b[0m\n"); + } else { + fprintf(stdout, "\n"); + } +} + +// prints a formatted message to stderr, color coded to red +void err(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 31, args); + va_end(args); + exit(1); +} + +// prints a formatted message to stderr, color coded to yellow +void warn(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 33, args); + va_end(args); +} + +// prints a formatted message to stderr, color coded to green +void info(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 32, args); + va_end(args); +} + +// prints a formatted message to stderr, color coded to white +void debug(const char* format, ...) { + va_list args; + va_start(args, format); + msg(format, 37, args); + va_end(args); +} + +// application main method +int main(int argc, char *argv[]) { + // check if the output is a tty so we can use colors + usecolor = isatty(1); + + static struct option long_options[] = { + {"color", no_argument, 0, 'c'}, + {0,0,0,0} + }; + + while (1) { + int c = getopt_long(argc, argv, "c", long_options, 0); + + if (c == -1) + break; + + switch (c) { + case 'c': + usecolor = true; + break; + default: + exit(1); + } + } + + // check for proper command line args + if(optind != argc-1) + err("usage: %s [--color] file.osm.pbf", argv[0]); + + // open specified file + FILE *fp = fopen(argv[optind], "r"); + + // read while the file has not reached its end + while(!feof(fp)) { + // storage of size, used multiple times + int32_t sz; + + // read the first 4 bytes of the file, this is the size of the blob-header + if(fread(&sz, sizeof(sz), 1, fp) != 1) + break; // end of file reached + + // convert the size from network byte-order to host byte-order + sz = ntohl(sz); + + // ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE + if(sz > OSMPBF::max_blob_header_size) + err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size); + + // read the blob-header from the file + if(fread(buffer, sz, 1, fp) != 1) + err("unable to read blob-header from file"); + + // parse the blob-header from the read-buffer + if(!blobheader.ParseFromArray(buffer, sz)) + err("unable to parse blob header"); + + // tell about the blob-header + info("\nBlobHeader (%d bytes)", sz); + debug(" type = %s", blobheader.type().c_str()); + + // size of the following blob + sz = blobheader.datasize(); + debug(" datasize = %u", sz); + + // optional indexdata + if(blobheader.has_indexdata()) + debug(" indexdata = %u bytes", blobheader.indexdata().size()); + + // ensure the blob is smaller then MAX_BLOB_SIZE + if(sz > OSMPBF::max_uncompressed_blob_size) + err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size); + + // read the blob from the file + if(fread(buffer, sz, 1, fp) != 1) + err("unable to read blob from file"); + + // parse the blob from the read-buffer + if(!blob.ParseFromArray(buffer, sz)) + err("unable to parse blob"); + + // tell about the blob-header + info("Blob (%d bytes)", sz); + + // set when we find at least one data stream + bool found_data = false; + + // if the blob has uncompressed data + if(blob.has_raw()) { + // we have at least one datastream + found_data = true; + + // size of the blob-data + sz = blob.raw().size(); + + // check that raw_size is set correctly + if(sz != blob.raw_size()) + warn(" reports wrong raw_size: %u bytes", blob.raw_size()); + + // tell about the blob-data + debug(" contains uncompressed data: %u bytes", sz); + + // copy the uncompressed data over to the unpack_buffer + memcpy(unpack_buffer, buffer, sz); + } + + // if the blob has zlib-compressed data + if(blob.has_zlib_data()) { + // issue a warning if there is more than one data steam, a blob may only contain one data stream + if(found_data) + warn(" contains several data streams"); + + // we have at least one datastream + found_data = true; + + // the size of the compressesd data + sz = blob.zlib_data().size(); + + // tell about the compressed data + debug(" contains zlib-compressed data: %u bytes", sz); + debug(" uncompressed size: %u bytes", blob.raw_size()); + + // zlib information + z_stream z; + + // next byte to decompress + z.next_in = (unsigned char*) blob.zlib_data().c_str(); + + // number of bytes to decompress + z.avail_in = sz; + + // place of next decompressed byte + z.next_out = (unsigned char*) unpack_buffer; + + // space for decompressed data + z.avail_out = blob.raw_size(); + + // misc + z.zalloc = Z_NULL; + z.zfree = Z_NULL; + z.opaque = Z_NULL; + + if(inflateInit(&z) != Z_OK) { + err(" failed to init zlib stream"); + } + if(inflate(&z, Z_FINISH) != Z_STREAM_END) { + err(" failed to inflate zlib stream"); + } + if(inflateEnd(&z) != Z_OK) { + err(" failed to deinit zlib stream"); + } + + // unpacked size + sz = z.total_out; + } + + // if the blob has lzma-compressed data + if(blob.has_lzma_data()) { + // issue a warning if there is more than one data steam, a blob may only contain one data stream + if(found_data) + warn(" contains several data streams"); + + // we have at least one datastream + found_data = true; + + // tell about the compressed data + debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size()); + debug(" uncompressed size: %u bytes", blob.raw_size()); + + // issue a warning, lzma compression is not yet supported + err(" lzma-decompression is not supported"); + } + + // check we have at least one data-stream + if(!found_data) + err(" does not contain any known data stream"); + + // switch between different blob-types + if(blobheader.type() == "OSMHeader") { + // tell about the OSMHeader blob + info(" OSMHeader"); + + // parse the HeaderBlock from the blob + if(!headerblock.ParseFromArray(unpack_buffer, sz)) + err("unable to parse header block"); + + // tell about the bbox + if(headerblock.has_bbox()) { + OSMPBF::HeaderBBox bbox = headerblock.bbox(); + debug(" bbox: %.7f,%.7f,%.7f,%.7f", + (double)bbox.left() / OSMPBF::lonlat_resolution, + (double)bbox.bottom() / OSMPBF::lonlat_resolution, + (double)bbox.right() / OSMPBF::lonlat_resolution, + (double)bbox.top() / OSMPBF::lonlat_resolution); + } + + // tell about the required features + for(int i = 0, l = headerblock.required_features_size(); i < l; i++) + debug(" required_feature: %s", headerblock.required_features(i).c_str()); + + // tell about the optional features + for(int i = 0, l = headerblock.optional_features_size(); i < l; i++) + debug(" required_feature: %s", headerblock.optional_features(i).c_str()); + + // tell about the writing program + if(headerblock.has_writingprogram()); + debug(" writingprogram: %s", headerblock.writingprogram().c_str()); + + // tell about the source + if(headerblock.has_source()) + debug(" source: %s", headerblock.source().c_str()); + } + + else if(blobheader.type() == "OSMData") { + // tell about the OSMData blob + info(" OSMData"); + + // parse the PrimitiveBlock from the blob + if(!primblock.ParseFromArray(unpack_buffer, sz)) + err("unable to parse primitive block"); + + // tell about the block's meta info + debug(" granularity: %u", primblock.granularity()); + debug(" lat_offset: %u", primblock.lat_offset()); + debug(" lon_offset: %u", primblock.lon_offset()); + debug(" date_granularity: %u", primblock.date_granularity()); + + // tell about the stringtable + debug(" stringtable: %u items", primblock.stringtable().s_size()); + + // number of PrimitiveGroups + debug(" primitivegroups: %u groups", primblock.primitivegroup_size()); + + // iterate over all PrimitiveGroups + for(int i = 0, l = primblock.primitivegroup_size(); i < l; i++) { + // one PrimitiveGroup from the the Block + OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i); + + bool found_items=false; + + // tell about nodes + if(pg.nodes_size() > 0) { + found_items = true; + + debug(" nodes: %d", pg.nodes_size()); + if(pg.nodes(0).has_info()) + debug(" with meta-info"); + } + + // tell about dense nodes + if(pg.has_dense()) { + found_items = true; + + debug(" dense nodes: %d", pg.dense().id_size()); + if(pg.dense().has_denseinfo()) + debug(" with meta-info"); + } + + // tell about ways + if(pg.ways_size() > 0) { + found_items = true; + + debug(" ways: %d", pg.ways_size()); + if(pg.ways(0).has_info()) + debug(" with meta-info"); + } + + // tell about relations + if(pg.relations_size() > 0) { + found_items = true; + + debug(" relations: %d", pg.relations_size()); + if(pg.relations(0).has_info()) + debug(" with meta-info"); + } + + if(!found_items) + warn(" contains no items"); + } + } + + else { + // unknown blob type + warn(" unknown blob type: %s", blobheader.type().c_str()); + } + } + + // close the file pointer + fclose(fp); + + // clean up the protobuf lib + google::protobuf::ShutdownProtobufLibrary(); +} +