From 46ea05fa121f7c5f412deaa2e65f21e49358960d Mon Sep 17 00:00:00 2001 From: Jonathan Ellis Date: Sat, 5 Feb 2011 19:38:35 +0000 Subject: [PATCH] Make BBU.string validate input for the desired Charset patch by jbellis; reviewed by slebresne for CASSANDRA-2091 git-svn-id: https://svn.apache.org/repos/asf/cassandra/branches/cassandra-0.7@1067490 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/cassandra/cli/CliClient.java | 5 ++-- .../cassandra/db/HintedHandOffManager.java | 16 +++++++++---- .../cassandra/db/marshal/AsciiType.java | 10 +++++++- .../apache/cassandra/db/marshal/UTF8Type.java | 6 ++--- .../CollatingOrderPreservingPartitioner.java | 6 +++-- .../dht/OrderPreservingPartitioner.java | 11 +++++++-- .../cassandra/dht/RandomPartitioner.java | 11 ++++++++- .../cassandra/utils/ByteBufferUtil.java | 24 +++++++++---------- .../apache/cassandra/utils/FBUtilities.java | 5 ---- .../org/apache/cassandra/db/TableTest.java | 10 +++++++- .../cassandra/utils/FBUtilitiesTest.java | 3 ++- 11 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/java/org/apache/cassandra/cli/CliClient.java b/src/java/org/apache/cassandra/cli/CliClient.java index ee0c5a1ca0..6e65ddf364 100644 --- a/src/java/org/apache/cassandra/cli/CliClient.java +++ b/src/java/org/apache/cassandra/cli/CliClient.java @@ -19,6 +19,7 @@ package org.apache.cassandra.cli; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.util.*; import com.google.common.base.Charsets; @@ -936,7 +937,7 @@ public class CliClient extends CliUserHelp } private void executeList(Tree statement) - throws TException, InvalidRequestException, NotFoundException, IllegalAccessException, InstantiationException, NoSuchFieldException, UnavailableException, TimedOutException + throws TException, InvalidRequestException, NotFoundException, IllegalAccessException, InstantiationException, NoSuchFieldException, UnavailableException, TimedOutException, CharacterCodingException { if (!CliMain.isConnected() || !hasKeySpace()) return; @@ -1896,7 +1897,7 @@ public class CliClient extends CliUserHelp * @throws NoSuchFieldException - column not found */ private void printSliceList(CfDef columnFamilyDef, List slices) - throws NotFoundException, TException, IllegalAccessException, InstantiationException, NoSuchFieldException + throws NotFoundException, TException, IllegalAccessException, InstantiationException, NoSuchFieldException, CharacterCodingException { AbstractType validator; String columnFamilyName = columnFamilyDef.getName(); diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java index a208b281cc..89d59809da 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java @@ -23,6 +23,7 @@ import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; @@ -229,12 +230,17 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean int index = ByteBufferUtil.lastIndexOf(joined, SEPARATOR.getBytes()[0], joined.limit()); if (index == -1 || index < (joined.position() + 1)) - throw new RuntimeException("Corrupted hint name " + ByteBufferUtil.string(joined)); + throw new RuntimeException("Corrupted hint name " + ByteBufferUtil.bytesToHex(joined)); - return new String[] { - ByteBufferUtil.string(joined, joined.position(), index - joined.position()), - ByteBufferUtil.string(joined, index + 1, joined.limit() - (index + 1)) - }; + try + { + return new String[] { ByteBufferUtil.string(joined, joined.position(), index - joined.position()), + ByteBufferUtil.string(joined, index + 1, joined.limit() - (index + 1)) }; + } + catch (CharacterCodingException e) + { + throw new RuntimeException(e); + } } private int waitForSchemaAgreement(InetAddress endpoint) throws InterruptedException diff --git a/src/java/org/apache/cassandra/db/marshal/AsciiType.java b/src/java/org/apache/cassandra/db/marshal/AsciiType.java index fc3ce7c233..bc54b2fd52 100644 --- a/src/java/org/apache/cassandra/db/marshal/AsciiType.java +++ b/src/java/org/apache/cassandra/db/marshal/AsciiType.java @@ -22,6 +22,7 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import com.google.common.base.Charsets; @@ -36,7 +37,14 @@ public class AsciiType extends BytesType @Override public String getString(ByteBuffer bytes) { - return ByteBufferUtil.string(bytes, Charsets.US_ASCII); + try + { + return ByteBufferUtil.string(bytes, Charsets.US_ASCII); + } + catch (CharacterCodingException e) + { + throw new MarshalException("Invalid ascii bytes " + ByteBufferUtil.bytesToHex(bytes)); + } } public ByteBuffer fromString(String source) diff --git a/src/java/org/apache/cassandra/db/marshal/UTF8Type.java b/src/java/org/apache/cassandra/db/marshal/UTF8Type.java index ded4184ede..c4ee5b6d42 100644 --- a/src/java/org/apache/cassandra/db/marshal/UTF8Type.java +++ b/src/java/org/apache/cassandra/db/marshal/UTF8Type.java @@ -22,12 +22,10 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; -import java.util.Arrays; import com.google.common.base.Charsets; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; public class UTF8Type extends BytesType { @@ -39,11 +37,11 @@ public class UTF8Type extends BytesType { try { - return FBUtilities.decodeToUTF8(bytes); + return ByteBufferUtil.string(bytes, Charsets.UTF_8); } catch (CharacterCodingException e) { - throw new MarshalException("invalid UTF8 bytes " + ByteBufferUtil.string(bytes)); + throw new MarshalException("invalid UTF8 bytes " + ByteBufferUtil.bytesToHex(bytes)); } } diff --git a/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java b/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java index 3f300f7cc1..5c890b55a5 100644 --- a/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java +++ b/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java @@ -25,7 +25,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import org.apache.cassandra.utils.FBUtilities; +import com.google.common.base.Charsets; + +import org.apache.cassandra.utils.ByteBufferUtil; public class CollatingOrderPreservingPartitioner extends AbstractByteOrderedPartitioner { @@ -39,7 +41,7 @@ public class CollatingOrderPreservingPartitioner extends AbstractByteOrderedPart String skey; try { - skey = FBUtilities.decodeToUTF8(key); + skey = ByteBufferUtil.string(key, Charsets.UTF_8); } catch (CharacterCodingException e) { diff --git a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java index 232fe51582..75695bb269 100644 --- a/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java +++ b/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java @@ -123,7 +123,14 @@ public class OrderPreservingPartitioner implements IPartitioner public Token fromByteArray(ByteBuffer bytes) { - return new StringToken(ByteBufferUtil.string(bytes, Charsets.UTF_8)); + try + { + return new StringToken(ByteBufferUtil.string(bytes, Charsets.UTF_8)); + } + catch (CharacterCodingException e) + { + throw new RuntimeException(e); + } } public String toString(Token stringToken) @@ -152,7 +159,7 @@ public class OrderPreservingPartitioner implements IPartitioner String skey; try { - skey = FBUtilities.decodeToUTF8(key); + skey = ByteBufferUtil.string(key, Charsets.UTF_8); } catch (CharacterCodingException e) { diff --git a/src/java/org/apache/cassandra/dht/RandomPartitioner.java b/src/java/org/apache/cassandra/dht/RandomPartitioner.java index 97d5b67ae9..61bb674436 100644 --- a/src/java/org/apache/cassandra/dht/RandomPartitioner.java +++ b/src/java/org/apache/cassandra/dht/RandomPartitioner.java @@ -21,6 +21,7 @@ package org.apache.cassandra.dht; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.util.*; import org.apache.cassandra.db.DecoratedKey; @@ -61,7 +62,15 @@ public class RandomPartitioner implements IPartitioner assert splitPoint != -1; // and decode the token and key - String token = ByteBufferUtil.string(fromdisk, fromdisk.position(), splitPoint - fromdisk.position(), UTF_8); + String token = null; + try + { + token = ByteBufferUtil.string(fromdisk, fromdisk.position(), splitPoint - fromdisk.position(), UTF_8); + } + catch (CharacterCodingException e) + { + throw new RuntimeException(e); + } ByteBuffer key = fromdisk.duplicate(); key.position(splitPoint + 1); return new DecoratedKey(new BigIntegerToken(token), key); diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index 46f690f429..1646ffdf06 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; import java.util.Arrays; @@ -100,28 +101,27 @@ public class ByteBufferUtil return compareUnsigned(o1, ByteBuffer.wrap(o2)); } - public static String string(ByteBuffer buffer) + public static String string(ByteBuffer buffer) throws CharacterCodingException { return string(buffer, Charset.defaultCharset()); } - public static String string(ByteBuffer buffer, Charset charset) - { - return string(buffer, buffer.position(), buffer.remaining(), charset); - } - - public static String string(ByteBuffer buffer, int offset, int length) + public static String string(ByteBuffer buffer, int offset, int length) throws CharacterCodingException { return string(buffer, offset, length, Charset.defaultCharset()); } - public static String string(ByteBuffer buffer, int offset, int length, Charset charset) + public static String string(ByteBuffer buffer, int offset, int length, Charset charset) throws CharacterCodingException { - if (buffer.hasArray()) - return new String(buffer.array(), buffer.arrayOffset() + offset, length, charset); + ByteBuffer copy = buffer.duplicate(); + copy.position(buffer.position() + offset); + copy.limit(copy.position() + length); + return string(buffer, charset); + } - byte[] buff = getArray(buffer, offset, length); - return new String(buff, charset); + public static String string(ByteBuffer buffer, Charset charset) throws CharacterCodingException + { + return charset.newDecoder().decode(buffer.duplicate()).toString(); } /** diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 4b409b1a3d..84f2efa130 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -398,11 +398,6 @@ public class FBUtilities return utflen; } - public static String decodeToUTF8(ByteBuffer bytes) throws CharacterCodingException - { - return Charsets.UTF_8.newDecoder().decode(bytes.duplicate()).toString(); - } - public static ByteBuffer toByteBuffer(long n) { byte[] bytes = new byte[8]; diff --git a/test/unit/org/apache/cassandra/db/TableTest.java b/test/unit/org/apache/cassandra/db/TableTest.java index 55adab5212..aa68ecb833 100644 --- a/test/unit/org/apache/cassandra/db/TableTest.java +++ b/test/unit/org/apache/cassandra/db/TableTest.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.*; @@ -511,7 +512,14 @@ public class TableTest extends CleanupHelper List L = new ArrayList(); for (IColumn column : columns) { - L.add(ByteBufferUtil.string(column.name())); + try + { + L.add(ByteBufferUtil.string(column.name())); + } + catch (CharacterCodingException e) + { + throw new AssertionError(e); + } } List names = new ArrayList(columnNames.length); diff --git a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java index 2b9da223eb..ea4408d0a1 100644 --- a/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java +++ b/test/unit/org/apache/cassandra/utils/FBUtilitiesTest.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.Arrays; +import com.google.common.base.Charsets; import org.junit.Test; public class FBUtilitiesTest @@ -89,6 +90,6 @@ public class FBUtilitiesTest public void testDecode() throws IOException { ByteBuffer bytes = ByteBuffer.wrap(new byte[]{(byte)0xff, (byte)0xfe}); - FBUtilities.decodeToUTF8(bytes); + ByteBufferUtil.string(bytes, Charsets.UTF_8); } }