This commit is contained in:
GitHub Merge Button 2011-10-16 06:56:27 -07:00
commit f9b417489b
14 changed files with 546 additions and 5 deletions

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
include/osmpbf/*.pb.h
src/*.pb.h
src/*.pb.o
src/libosmpbf.a
*.swp

27
README
View File

@ -4,6 +4,29 @@ OSM PBF
See http://wiki.openstreetmap.org/wiki/PBF_Format .
To build the Debian/Ubuntu package call:
debuild -I -us -uc
There is a Java and a C version of the PBF library code here.
C Version
=========
To compile:
cd src
make
To install:
cd src
make install
To build the Debian/Ubuntu package call:
debuild -I -us -uc
To install the Debian/Ubuntu package call:
sudo dpkg --install ../libosmpbf-dev*.deb
To include in your program use:
#include <osmpbf/osmpbf.h>
and link with:
-lpthread -lz -lprotobuf-lite -losmpbf

14
debian/changelog vendored
View File

@ -1,3 +1,17 @@
libosmpbf-dev (1.1.1j2) maverick; urgency=low
* Add manpage for osmpbf-outline tool
-- Jochen Topf <jochen@topf.org> Fri, 6 Jun 2011 10:30:12 +0100
libosmpbf-dev (1.1.1j1) maverick; urgency=low
* Add support for OSM history file
* Add some often used constants to OSMPBF namespace
* Add osmpbf-outline testing tool
-- Jochen Topf <jochen@topf.org> Fri, 3 Jun 2011 11:36:17 +0100
libosmpbf-dev (0.1) maverick; urgency=low
* Initial debian package

1
debian/docs vendored
View File

@ -1,2 +1,3 @@
src/fileformat.proto
src/osmformat.proto
README

1
debian/libosmpbf-dev.manpages vendored Normal file
View File

@ -0,0 +1 @@
tools/osmpbf-outline.1

3
debian/rules vendored
View File

@ -24,6 +24,7 @@ build-stamp: configure-stamp
# Add here commands to compile the package.
$(MAKE) -C src
$(MAKE) -C tools
touch $@
@ -34,6 +35,7 @@ clean:
# Add here commands to clean up after the build process.
$(MAKE) -C src clean || /bin/true
$(MAKE) -C tools clean || /bin/true
dh_clean
@ -45,6 +47,7 @@ install: build
# Add here commands to install the package into debian/libosmpbf-dev.
$(MAKE) -C src DESTDIR=$(CURDIR)/debian/libosmpbf-dev install
$(MAKE) -C tools DESTDIR=$(CURDIR)/debian/libosmpbf-dev install
# Build architecture-independent files here.

27
include/osmpbf/osmpbf.h Normal file
View File

@ -0,0 +1,27 @@
#ifndef OSMPBF_H
#define OSMPBF_H
// this describes the low-level blob storage
#include <osmpbf/fileformat.pb.h>
// this describes the high-level OSM objects
#include <osmpbf/osmformat.pb.h>
namespace OSMPBF {
// library version
const char *version = "1.1.1j1";
// the maximum size of a blob header in bytes
const int max_blob_header_size = 64 * 1024; // 64 kB
// the maximum size of an uncompressed blob in bytes
const int max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB
// resolution for longitude/latitude used for conversion
// between representation as double and as int
const int lonlat_resolution = 1000 * 1000 * 1000;
}
#endif // OSMPBF_H

3
magic Normal file
View File

@ -0,0 +1,3 @@
# Magic data for file(1) command.
# Format is described in magic(5).
6 string OSMHeader OpenStreetMap PBF

View File

@ -3,7 +3,7 @@ CXX = g++
CXXFLAGS = -O3
AR = ar
all: libosmpbf.a fileformat.pb.h osmformat.pb.h
all: libosmpbf.a ../include/osmpbf/fileformat.pb.h ../include/osmpbf/osmformat.pb.h
libosmpbf.a: fileformat.pb.o osmformat.pb.o
$(AR) -cr $@ fileformat.pb.o osmformat.pb.o
@ -11,14 +11,17 @@ libosmpbf.a: fileformat.pb.o osmformat.pb.o
%.pb.o: %.pb.cc
$(CXX) $(CXXFLAGS) -c -o $@ $<
%.pb.cc %.pb.h: %.proto
%.pb.cc ../include/osmpbf/%.pb.h: %.proto
protoc --proto_path=. --cpp_out=. $<
cp *.pb.h ../include/osmpbf/
install:
install -m 755 -g root -o root -d $(DESTDIR)/usr/lib
install -m 644 -g root -o root libosmpbf.a $(DESTDIR)/usr/lib
install -m 755 -g root -o root -d $(DESTDIR)/usr/include/osmpbf
install -m 644 -g root -o root fileformat.pb.h osmformat.pb.h $(DESTDIR)/usr/include/osmpbf
install -m 644 -g root -o root ../include/osmpbf/osmpbf.h $(DESTDIR)/usr/include/osmpbf
install -m 644 -g root -o root ../include/osmpbf/fileformat.pb.h $(DESTDIR)/usr/include/osmpbf
install -m 644 -g root -o root ../include/osmpbf/osmformat.pb.h $(DESTDIR)/usr/include/osmpbf
clean:
rm -f *.pb.h *.pb.cc *.pb.o libosmpbf.a

View File

@ -126,6 +126,16 @@ message Info {
optional int64 changeset = 3;
optional int32 uid = 4;
optional uint32 user_sid = 5; // String IDs
// The visible flag is used to store history information. It indicates that
// the current object version has been created by a delete operation on the
// OSM API.
// When a writer sets this flag, it MUST add a required_features tag with
// value "HistoricalInformation" to the HeaderBlock.
// If this flag is not available for some object it MUST be assumed to be
// true if the file has the required_features tag "HistoricalInformation"
// set.
optional bool visible = 6;
}
/** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */
@ -135,6 +145,16 @@ message DenseInfo {
repeated sint64 changeset = 3 [packed = true]; // DELTA coded
repeated sint32 uid = 4 [packed = true]; // DELTA coded
repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded
// The visible flag is used to store history information. It indicates that
// the current object version has been created by a delete operation on the
// OSM API.
// When a writer sets this flag, it MUST add a required_features tag with
// value "HistoricalInformation" to the HeaderBlock.
// If this flag is not available for some object it MUST be assumed to be
// true if the file has the required_features tag "HistoricalInformation"
// set.
repeated bool visible = 6 [packed = true];
}

1
tools/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
osmpbf-outline

16
tools/Makefile Normal file
View File

@ -0,0 +1,16 @@
CXX = g++
CXXFLAGS = -g -I../include
LDFLAGS = -L../src -lpthread -lz -lprotobuf-lite -losmpbf
all: osmpbf-outline
osmpbf-outline: osmpbf-outline.cpp
$(CXX) $(CXXFLAGS) -o $@ $< $(LDFLAGS)
install:
install -m 755 -g root -o root -d $(DESTDIR)/usr/bin
install -m 644 -g root -o root -s osmpbf-outline $(DESTDIR)/usr/bin
clean:
rm -f osmpbf-outline

49
tools/osmpbf-outline.1 Normal file
View File

@ -0,0 +1,49 @@
.TH osmpbf-outline 1 LOCAL
.SH NAME
osmpbf-outline - outline the content of an .osm.pbf or .osh.pbf file
.SH SYNOPSIS
.B osmpbf-outline
[
.B --color
]
.I filename
.SH DESCRIPTION
.I osmpbf-outline
generates an colored outline an .osm.pbf or .osh.pbf file, revealing its inner structure.
It does not decode all information nor does it check for of possible mistakes, but it will
warn on some bad mistakes one can make when generating .osm.pbf files.
This tool can be used by authors of .osm.pbf readers and writers and is useful to understand
the inner blob/block structure of .osm.pbf files.
Its source code is heavily documented and can be used as a good start on how to read- and
write .osm.pbf files with C++.
.SH OPTIONS
.TP 5
.B -c --color
usually colorization is disabled when writing to a file descriptor that is no tty (eg. when
the output is piped to a file or to a pager like to more or less). Some pagers can handle ANSI
color code (
.I more
can, less can when invoked as
.I less -R
). To enforce colorization when working with such pagers, specify the
.B --color
flag.
.SH EXAMPLES
osmpbf-outline germany.osm.pbf
.SH AUTHOR
Peter Koerner <peter@mazdermind.de>
Jochen Topf <jochen@topf.org>

378
tools/osmpbf-outline.cpp Normal file
View File

@ -0,0 +1,378 @@
// used for va_list in debug-print methods
#include <stdarg.h>
// file io lib
#include <stdio.h>
// getopt is used to check for the --color-flag
#include <getopt.h>
// zlib compression is used inside the pbf blobs
#include <zlib.h>
// netinet provides the network-byte-order conversion function
#include <netinet/in.h>
// this is the header to pbf format
#include <osmpbf/osmpbf.h>
// should the output use color?
bool usecolor = false;
// buffer for reading a compressed blob from file
char buffer[OSMPBF::max_uncompressed_blob_size];
// buffer for decompressing the blob
char unpack_buffer[OSMPBF::max_uncompressed_blob_size];
// pbf struct of a BlobHeader
OSMPBF::BlobHeader blobheader;
// pbf struct of a Blob
OSMPBF::Blob blob;
// pbf struct of an OSM HeaderBlock
OSMPBF::HeaderBlock headerblock;
// pbf struct of an OSM PrimitiveBlock
OSMPBF::PrimitiveBlock primblock;
// prints a formatted message to stdout, optionally color coded
void msg(const char* format, int color, va_list args) {
if(usecolor) {
fprintf(stdout, "\x1b[0;%dm", color);
}
vfprintf(stdout, format, args);
if(usecolor) {
fprintf(stdout, "\x1b[0m\n");
} else {
fprintf(stdout, "\n");
}
}
// prints a formatted message to stderr, color coded to red
void err(const char* format, ...) {
va_list args;
va_start(args, format);
msg(format, 31, args);
va_end(args);
exit(1);
}
// prints a formatted message to stderr, color coded to yellow
void warn(const char* format, ...) {
va_list args;
va_start(args, format);
msg(format, 33, args);
va_end(args);
}
// prints a formatted message to stderr, color coded to green
void info(const char* format, ...) {
va_list args;
va_start(args, format);
msg(format, 32, args);
va_end(args);
}
// prints a formatted message to stderr, color coded to white
void debug(const char* format, ...) {
va_list args;
va_start(args, format);
msg(format, 37, args);
va_end(args);
}
// application main method
int main(int argc, char *argv[]) {
// check if the output is a tty so we can use colors
usecolor = isatty(1);
static struct option long_options[] = {
{"color", no_argument, 0, 'c'},
{0,0,0,0}
};
while (1) {
int c = getopt_long(argc, argv, "c", long_options, 0);
if (c == -1)
break;
switch (c) {
case 'c':
usecolor = true;
break;
default:
exit(1);
}
}
// check for proper command line args
if(optind != argc-1)
err("usage: %s [--color] file.osm.pbf", argv[0]);
// open specified file
FILE *fp = fopen(argv[optind], "r");
// read while the file has not reached its end
while(!feof(fp)) {
// storage of size, used multiple times
int32_t sz;
// read the first 4 bytes of the file, this is the size of the blob-header
if(fread(&sz, sizeof(sz), 1, fp) != 1)
break; // end of file reached
// convert the size from network byte-order to host byte-order
sz = ntohl(sz);
// ensure the blob-header is smaller then MAX_BLOB_HEADER_SIZE
if(sz > OSMPBF::max_blob_header_size)
err("blob-header-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_blob_header_size);
// read the blob-header from the file
if(fread(buffer, sz, 1, fp) != 1)
err("unable to read blob-header from file");
// parse the blob-header from the read-buffer
if(!blobheader.ParseFromArray(buffer, sz))
err("unable to parse blob header");
// tell about the blob-header
info("\nBlobHeader (%d bytes)", sz);
debug(" type = %s", blobheader.type().c_str());
// size of the following blob
sz = blobheader.datasize();
debug(" datasize = %u", sz);
// optional indexdata
if(blobheader.has_indexdata())
debug(" indexdata = %u bytes", blobheader.indexdata().size());
// ensure the blob is smaller then MAX_BLOB_SIZE
if(sz > OSMPBF::max_uncompressed_blob_size)
err("blob-size is bigger then allowed (%u > %u)", sz, OSMPBF::max_uncompressed_blob_size);
// read the blob from the file
if(fread(buffer, sz, 1, fp) != 1)
err("unable to read blob from file");
// parse the blob from the read-buffer
if(!blob.ParseFromArray(buffer, sz))
err("unable to parse blob");
// tell about the blob-header
info("Blob (%d bytes)", sz);
// set when we find at least one data stream
bool found_data = false;
// if the blob has uncompressed data
if(blob.has_raw()) {
// we have at least one datastream
found_data = true;
// size of the blob-data
sz = blob.raw().size();
// check that raw_size is set correctly
if(sz != blob.raw_size())
warn(" reports wrong raw_size: %u bytes", blob.raw_size());
// tell about the blob-data
debug(" contains uncompressed data: %u bytes", sz);
// copy the uncompressed data over to the unpack_buffer
memcpy(unpack_buffer, buffer, sz);
}
// if the blob has zlib-compressed data
if(blob.has_zlib_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if(found_data)
warn(" contains several data streams");
// we have at least one datastream
found_data = true;
// the size of the compressesd data
sz = blob.zlib_data().size();
// tell about the compressed data
debug(" contains zlib-compressed data: %u bytes", sz);
debug(" uncompressed size: %u bytes", blob.raw_size());
// zlib information
z_stream z;
// next byte to decompress
z.next_in = (unsigned char*) blob.zlib_data().c_str();
// number of bytes to decompress
z.avail_in = sz;
// place of next decompressed byte
z.next_out = (unsigned char*) unpack_buffer;
// space for decompressed data
z.avail_out = blob.raw_size();
// misc
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
if(inflateInit(&z) != Z_OK) {
err(" failed to init zlib stream");
}
if(inflate(&z, Z_FINISH) != Z_STREAM_END) {
err(" failed to inflate zlib stream");
}
if(inflateEnd(&z) != Z_OK) {
err(" failed to deinit zlib stream");
}
// unpacked size
sz = z.total_out;
}
// if the blob has lzma-compressed data
if(blob.has_lzma_data()) {
// issue a warning if there is more than one data steam, a blob may only contain one data stream
if(found_data)
warn(" contains several data streams");
// we have at least one datastream
found_data = true;
// tell about the compressed data
debug(" contains lzma-compressed data: %u bytes", blob.lzma_data().size());
debug(" uncompressed size: %u bytes", blob.raw_size());
// issue a warning, lzma compression is not yet supported
err(" lzma-decompression is not supported");
}
// check we have at least one data-stream
if(!found_data)
err(" does not contain any known data stream");
// switch between different blob-types
if(blobheader.type() == "OSMHeader") {
// tell about the OSMHeader blob
info(" OSMHeader");
// parse the HeaderBlock from the blob
if(!headerblock.ParseFromArray(unpack_buffer, sz))
err("unable to parse header block");
// tell about the bbox
if(headerblock.has_bbox()) {
OSMPBF::HeaderBBox bbox = headerblock.bbox();
debug(" bbox: %.7f,%.7f,%.7f,%.7f",
(double)bbox.left() / OSMPBF::lonlat_resolution,
(double)bbox.bottom() / OSMPBF::lonlat_resolution,
(double)bbox.right() / OSMPBF::lonlat_resolution,
(double)bbox.top() / OSMPBF::lonlat_resolution);
}
// tell about the required features
for(int i = 0, l = headerblock.required_features_size(); i < l; i++)
debug(" required_feature: %s", headerblock.required_features(i).c_str());
// tell about the optional features
for(int i = 0, l = headerblock.optional_features_size(); i < l; i++)
debug(" required_feature: %s", headerblock.optional_features(i).c_str());
// tell about the writing program
if(headerblock.has_writingprogram());
debug(" writingprogram: %s", headerblock.writingprogram().c_str());
// tell about the source
if(headerblock.has_source())
debug(" source: %s", headerblock.source().c_str());
}
else if(blobheader.type() == "OSMData") {
// tell about the OSMData blob
info(" OSMData");
// parse the PrimitiveBlock from the blob
if(!primblock.ParseFromArray(unpack_buffer, sz))
err("unable to parse primitive block");
// tell about the block's meta info
debug(" granularity: %u", primblock.granularity());
debug(" lat_offset: %u", primblock.lat_offset());
debug(" lon_offset: %u", primblock.lon_offset());
debug(" date_granularity: %u", primblock.date_granularity());
// tell about the stringtable
debug(" stringtable: %u items", primblock.stringtable().s_size());
// number of PrimitiveGroups
debug(" primitivegroups: %u groups", primblock.primitivegroup_size());
// iterate over all PrimitiveGroups
for(int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
// one PrimitiveGroup from the the Block
OSMPBF::PrimitiveGroup pg = primblock.primitivegroup(i);
bool found_items=false;
// tell about nodes
if(pg.nodes_size() > 0) {
found_items = true;
debug(" nodes: %d", pg.nodes_size());
if(pg.nodes(0).has_info())
debug(" with meta-info");
}
// tell about dense nodes
if(pg.has_dense()) {
found_items = true;
debug(" dense nodes: %d", pg.dense().id_size());
if(pg.dense().has_denseinfo())
debug(" with meta-info");
}
// tell about ways
if(pg.ways_size() > 0) {
found_items = true;
debug(" ways: %d", pg.ways_size());
if(pg.ways(0).has_info())
debug(" with meta-info");
}
// tell about relations
if(pg.relations_size() > 0) {
found_items = true;
debug(" relations: %d", pg.relations_size());
if(pg.relations(0).has_info())
debug(" with meta-info");
}
if(!found_items)
warn(" contains no items");
}
}
else {
// unknown blob type
warn(" unknown blob type: %s", blobheader.type().c_str());
}
}
// close the file pointer
fclose(fp);
// clean up the protobuf lib
google::protobuf::ShutdownProtobufLibrary();
}