natas_hw
/
OSM-binary
mirror of https://github.com/openstreetmap/OSM-binary
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #60 from simon04/java-8 

Modernise Java codebase
This commit is contained in:
Jochen Topf 2021-01-04 09:16:56 +01:00 committed by GitHub
parent bf2076840e d5bdd020c1
commit ee9144ef25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 63 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src.java/crosby/binary/BinaryParser.java
View File

@ -18,27 +18,28 @@
package crosby.binary;
import java.io.UncheckedIOException;
import java.util.Date;
import java.util.List;
import com.google.protobuf.InvalidProtocolBufferException;
import crosby.binary.Osmformat;
import crosby.binary.file.BlockReaderAdapter;
import crosby.binary.file.FileBlock;
import crosby.binary.file.FileBlockPosition;
import crosby.binary.file.FileFormatException;
public abstract class BinaryParser implements BlockReaderAdapter {
protected int granularity;
private long lat_offset;
private long lon_offset;
protected int date_granularity;
private String strings[];
private String[] strings;
/** Take a Info protocol buffer containing a date and convert it into a java Date object */
protected Date getDate(Osmformat.Info info) {
if (info.hasTimestamp()) {
return new Date(date_granularity * (long) info.getTimestamp());
return new Date(date_granularity * info.getTimestamp());
} else
return NODATE;
}
@ -47,8 +48,8 @@ public abstract class BinaryParser implements BlockReaderAdapter {
/** Get a string based on the index used.
*
* Index 0 is reserved to use as a delimiter, therefore, index 1 corresponds to the first string in the table
* @param id
* @return
* @param id the index
* @return the string at the given index
*/
protected String getStringById(int id) {
return strings[id];
@ -56,7 +57,6 @@ public abstract class BinaryParser implements BlockReaderAdapter {
@Override
public void handleBlock(FileBlock message) {
// TODO Auto-generated method stub
try {
if (message.getType().equals("OSMHeader")) {
Osmformat.HeaderBlock headerblock = Osmformat.HeaderBlock
@ -68,9 +68,7 @@ public abstract class BinaryParser implements BlockReaderAdapter {
parse(primblock);
}
} catch (InvalidProtocolBufferException e) {
// TODO Auto-generated catch block
e.printStackTrace();
throw new Error("ParseError"); // TODO
throw new UncheckedIOException(new FileFormatException(e));
}
}

19
src.java/crosby/binary/BinarySerializer.java
View File

@ -17,7 +17,10 @@
package crosby.binary;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.List;
@ -35,7 +38,7 @@ import crosby.binary.file.FileBlock;
* ordered to process their data at the appropriate time.
* */
public class BinarySerializer {
public class BinarySerializer implements Closeable, Flushable {
/**
* Interface used to write a group of primitives. One of these for each
@ -67,11 +70,11 @@ public class BinarySerializer {
this.batch_limit = batch_limit;
}
// Paramaters affecting the output size.
// Parameters affecting the output size.
protected final int MIN_DENSE = 10;
protected int batch_limit = 4000;
// Parmaters affecting the output.
// Parameters affecting the output.
protected int granularity = 100;
protected int date_granularity = 1000;
@ -80,8 +83,8 @@ public class BinarySerializer {
/** How many primitives have been seen in this batch */
protected int batch_size = 0;
protected int total_entities = 0;
private StringTable stringtable = new StringTable();
protected List<PrimGroupWriterInterface> groups = new ArrayList<PrimGroupWriterInterface>();
private final StringTable stringtable = new StringTable();
protected List<PrimGroupWriterInterface> groups = new ArrayList<>();
protected BlockOutputStream output;
public BinarySerializer(BlockOutputStream output) {
@ -92,11 +95,13 @@ public class BinarySerializer {
return stringtable;
}
@Override
public void flush() throws IOException {
processBatch();
output.flush();
}
@Override
public void close() throws IOException {
flush();
output.close();
@ -139,9 +144,7 @@ public class BinarySerializer {
output.write(FileBlock.newInstance("OSMData", message
.toByteString(), null));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
throw new Error(e);
throw new UncheckedIOException(e);
} finally {
batch_size = 0;
groups.clear();

43
src.java/crosby/binary/StringTable.java
View File

@ -24,7 +24,7 @@ import java.util.HashMap;
import com.google.protobuf.ByteString;
/**
* Class for mapping a set of strings to integers, giving frequently occuring
* Class for mapping a set of strings to integers, giving frequently occurring
* strings small integers.
*/
public class StringTable {
@ -34,34 +34,28 @@ public class StringTable {
private HashMap<String, Integer> counts;
private HashMap<String, Integer> stringmap;
private String set[];
private String[] set;
/**
* Increments the count of the given string
* @param s the string
*/
public void incr(String s) {
if (counts.containsKey(s)) {
counts.put(s, new Integer(counts.get(s).intValue() + 1));
} else {
counts.put(s, new Integer(1));
}
counts.merge(s, 1, Integer::sum);
}
/** After the stringtable has been built, return the offset of a string in it.
*
* Note, value '0' is reserved for use as a delimiter and will not be returned.
* @param s
* @return
* @param s the string to lookup
* @return the offset of the string
*/
public int getIndex(String s) {
return stringmap.get(s).intValue();
return stringmap.get(s);
}
public void finish() {
Comparator<String> comparator = new Comparator<String>() {
@Override
public int compare(final String s1, String s2) {
int diff = counts.get(s2) - counts.get(s1);
return diff;
}
};
Comparator<String> comparator = (s1, s2) -> counts.get(s2) - counts.get(s1);
/* Sort the stringtable */
@ -88,7 +82,7 @@ public class StringTable {
So, when I decide on the master stringtable to use, I put the 127 most frequently occurring
strings into A (accomplishing goal 1), and sort them by frequency (to accomplish goal 2), but
for B and C, which contain the less progressively less frequently encountered strings, I sort
them lexiconographically, to maximize goal 3 and ignoring goal 2.
them lexicographically, to maximize goal 3 and ignoring goal 2.
Goal 1 is the most important. Goal 2 helped enough to be worth it, and goal 3 was pretty minor,
but all should be re-benchmarked.
@ -103,7 +97,7 @@ public class StringTable {
// Sort based on the frequency.
Arrays.sort(set, comparator);
// Each group of keys that serializes to the same number of bytes is
// sorted lexiconographically.
// sorted lexicographically.
// to maximize deflate compression.
// Don't sort the first array. There's not likely to be much benefit, and we want frequent values to be small.
@ -114,15 +108,15 @@ public class StringTable {
Arrays.sort(set, Math.min(1 << 14, set.length-1), Math.min(1 << 21,
set.length-1), comparator);
}
stringmap = new HashMap<String, Integer>(2 * set.length);
stringmap = new HashMap<>(2 * set.length);
for (int i = 0; i < set.length; i++) {
stringmap.put(set[i], new Integer(i+1)); // Index 0 is reserved for use as a delimiter.
stringmap.put(set[i], i + 1); // Index 0 is reserved for use as a delimiter.
}
counts = null;
}
public void clear() {
counts = new HashMap<String, Integer>(100);
counts = new HashMap<>(100);
stringmap = null;
set = null;
}
@ -131,8 +125,9 @@ public class StringTable {
Osmformat.StringTable.Builder builder = Osmformat.StringTable
.newBuilder();
builder.addS(ByteString.copyFromUtf8("")); // Add a unused string at offset 0 which is used as a delimiter.
for (int i = 0; i < set.length; i++)
builder.addS(ByteString.copyFromUtf8(set[i]));
for (String s : set) {
builder.addS(ByteString.copyFromUtf8(s));
}
return builder;
}
}

4
src.java/crosby/binary/file/BlockInputStream.java
View File

@ -17,11 +17,12 @@
package crosby.binary.file;
import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
public class BlockInputStream {
public class BlockInputStream implements Closeable {
// TODO: Should be seekable input stream!
public BlockInputStream(InputStream input, BlockReaderAdapter adaptor) {
this.input = input;
@ -38,6 +39,7 @@ public class BlockInputStream {
}
}
@Override
public void close() throws IOException {
input.close();
}

4
src.java/crosby/binary/file/BlockOutputStream.java
View File

@ -44,7 +44,7 @@ public class BlockOutputStream {
else if (s.equals("deflate"))
compression = CompressFlags.DEFLATE;
else
throw new Error("Unknown compression type: " + s);
throw new IllegalArgumentException("Unknown compression type: " + s);
}
/** Write a block with the stream's default compression flag */
@ -69,6 +69,6 @@ public class BlockOutputStream {
}
OutputStream outwrite;
List<FileBlockPosition> writtenblocks = new ArrayList<FileBlockPosition>();
List<FileBlockPosition> writtenblocks = new ArrayList<>();
CompressFlags compression;
}

12
src.java/crosby/binary/file/FileBlock.java
View File

@ -18,10 +18,12 @@
package crosby.binary.file;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.zip.Deflater;
@ -48,13 +50,13 @@ public class FileBlock extends FileBlockBase {
if (blob != null && blob.size() > MAX_BODY_SIZE/2) {
System.err.println("Warning: Fileblock has body size too large and may be considered corrupt");
if (blob != null && blob.size() > MAX_BODY_SIZE-1024*1024) {
throw new Error("This file has too many entities in a block. Parsers will reject it.");
throw new IllegalArgumentException("This file has too many entities in a block. Parsers will reject it.");
}
}
if (indexdata != null && indexdata.size() > MAX_HEADER_SIZE/2) {
System.err.println("Warning: Fileblock has indexdata too large and may be considered corrupt");
if (indexdata != null && indexdata.size() > MAX_HEADER_SIZE-512) {
throw new Error("This file header is too large. Parsers will reject it.");
throw new IllegalArgumentException("This file header is too large. Parsers will reject it.");
}
}
return new FileBlock(type, blob, indexdata);
@ -65,7 +67,7 @@ public class FileBlock extends FileBlockBase {
Deflater deflater = new Deflater();
deflater.setInput(data.toByteArray());
deflater.finish();
byte out[] = new byte[size];
byte[] out = new byte[size];
deflater.deflate(out);
if (!deflater.finished()) {
@ -77,7 +79,7 @@ public class FileBlock extends FileBlockBase {
deflater.deflate(out, deflater.getTotalOut(), out.length
- deflater.getTotalOut());
if (!deflater.finished()) {
throw new Error("Internal error in compressor");
throw new UncheckedIOException(new EOFException());
}
}
ByteString compressed = ByteString.copyFrom(out, 0, deflater
@ -103,7 +105,7 @@ public class FileBlock extends FileBlockBase {
if (flags == CompressFlags.DEFLATE)
deflateInto(blobbuilder);
else
throw new Error("Compression flag not understood");
throw new IllegalArgumentException("Compression flag not understood");
}
Fileformat.Blob blob = blobbuilder.build();

4
src.java/crosby/binary/file/FileBlockHead.java
View File

@ -50,7 +50,7 @@ public class FileBlockHead extends FileBlockReference {
throw new FileFormatException("Unexpectedly long header "+MAX_HEADER_SIZE+ " bytes. Possibly corrupt file.");
}
byte buf[] = new byte[headersize];
byte[] buf = new byte[headersize];
datinput.readFully(buf);
// System.out.format("Read buffer for header of %d bytes\n",buf.length);
Fileformat.BlobHeader header = Fileformat.BlobHeader
@ -90,7 +90,7 @@ public class FileBlockHead extends FileBlockReference {
*/
FileBlock readContents(InputStream input) throws IOException {
DataInputStream datinput = new DataInputStream(input);
byte buf[] = new byte[getDatasize()];
byte[] buf = new byte[getDatasize()];
datinput.readFully(buf);
return parseData(buf);
}

16
src.java/crosby/binary/file/FileBlockPosition.java
View File

@ -21,6 +21,7 @@ import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
@ -43,21 +44,20 @@ public class FileBlockPosition extends FileBlockBase {
}
/** Parse out and decompress the data part of a fileblock helper function. */
FileBlock parseData(byte buf[]) throws InvalidProtocolBufferException {
FileBlock parseData(byte[] buf) throws InvalidProtocolBufferException {
FileBlock out = FileBlock.newInstance(type, null, indexdata);
Fileformat.Blob blob = Fileformat.Blob.parseFrom(buf);
if (blob.hasRaw()) {
out.data = blob.getRaw();
} else if (blob.hasZlibData()) {
byte buf2[] = new byte[blob.getRawSize()];
byte[] buf2 = new byte[blob.getRawSize()];
Inflater decompresser = new Inflater();
decompresser.setInput(blob.getZlibData().toByteArray());
// decompresser.getRemaining();
try {
decompresser.inflate(buf2);
} catch (DataFormatException e) {
e.printStackTrace();
throw new Error(e);
throw new UncheckedIOException(new FileFormatException(e));
}
assert (decompresser.finished());
decompresser.end();
@ -85,11 +85,11 @@ public class FileBlockPosition extends FileBlockBase {
public FileBlock read(InputStream input) throws IOException {
if (input instanceof FileInputStream) {
((FileInputStream) input).getChannel().position(data_offset);
byte buf[] = new byte[getDatasize()];
byte[] buf = new byte[getDatasize()];
(new DataInputStream(input)).readFully(buf);
return parseData(buf);
} else {
throw new Error("Random access binary reads require seekability");
throw new IllegalArgumentException("Random access binary reads require seekability");
}
}
@ -98,12 +98,12 @@ public class FileBlockPosition extends FileBlockBase {
* stored.
*/
public ByteString serialize() {
throw new Error("TODO");
throw new UnsupportedOperationException("TODO");
}
/** TODO: Parse a serialized representation of this block reference */
static FileBlockPosition parseFrom(ByteString b) {
throw new Error("TODO");
throw new UnsupportedOperationException("TODO");
}
protected int datasize;

4
src.java/crosby/binary/file/FileFormatException.java
View File

@ -25,6 +25,10 @@ public class FileFormatException extends IOException {
super(string);
}
public FileFormatException(Throwable cause) {
super(cause);
}
/**
*
*/