jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!660 Supporting Char Type in getting Bucket Hash Code 

Merge pull request !660 from Surya Sumanth/char_support_fix
This commit is contained in:
i-robot 2021-03-05 21:18:25 +08:00 committed by Gitee
parent 517658adfe 5429a472c5
commit 3cecc221d2
3 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketingV1.java
View File

@ -19,6 +19,7 @@ import io.airlift.slice.Slice;
import io.prestosql.plugin.hive.HiveType;
import io.prestosql.spi.Page;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.type.Chars;
import io.prestosql.spi.type.Type;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@ -103,6 +104,8 @@ public final class HiveBucketingV1
return hashBytes(0, prestoType.getSlice(block, position));
case VARCHAR:
return hashBytes(1, prestoType.getSlice(block, position));
case CHAR:
return hashBytes(1, Chars.truncateToLengthAndTrimSpaces(prestoType.getSlice(block, position), prestoType));
case DATE:
// day offset from 1970-01-01
return toIntExact(prestoType.getLong(block, position));
@ -153,6 +156,8 @@ public final class HiveBucketingV1
return hashBytes(0, (Slice) value);
case VARCHAR:
return hashBytes(1, (Slice) value);
case CHAR:
return hashBytes(1, (Slice) value);
case DATE:
// day offset from 1970-01-01
return toIntExact((long) value);

5
presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketingV2.java
View File

@ -19,6 +19,7 @@ import io.airlift.slice.Slice;
import io.prestosql.plugin.hive.HiveType;
import io.prestosql.spi.Page;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.type.Chars;
import io.prestosql.spi.type.Type;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@ -108,6 +109,8 @@ public final class HiveBucketingV2
return Murmur3.hash32(prestoType.getSlice(block, position).getBytes());
case VARCHAR:
return Murmur3.hash32(prestoType.getSlice(block, position).getBytes());
case CHAR:
return Murmur3.hash32(Chars.truncateToLengthAndTrimSpaces(prestoType.getSlice(block, position), prestoType).getBytes());
case DATE:
// day offset from 1970-01-01
return Murmur3.hash32(bytes(toIntExact(prestoType.getLong(block, position))));
@ -159,6 +162,8 @@ public final class HiveBucketingV2
return Murmur3.hash32(((Slice) value).getBytes());
case VARCHAR:
return Murmur3.hash32(((Slice) value).getBytes());
case CHAR:
return Murmur3.hash32(((Slice) value).getBytes());
case DATE:
// day offset from 1970-01-01
return Murmur3.hash32(bytes(toIntExact((long) value)));

6
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveBucketing.java
View File

@ -112,6 +112,10 @@ public class TestHiveBucketing
assertBucketEquals("string", "\u5f3a\u5927\u7684Hetu\u5f15\u64ce", 889847277, 1436831192); // 3-byte UTF-8 sequences (in Basic Plane, i.e. Plane 0)
assertBucketEquals("string", "\uD843\uDFFC\uD843\uDFFD\uD843\uDFFE\uD843\uDFFF", -1810797254, -697348811); // 4 code points: 20FFC - 20FFF. 4-byte UTF-8 sequences in Supplementary Plane 2
assertBucketEquals("char(6)", null, 0, 0);
assertBucketEquals("char(6)", "", 1, -965378730);
assertBucketEquals("char(6)", "test_1", 10333957, 1284522943);
assertBucketEquals("date", null, 0, 0);
assertBucketEquals("date", Date.valueOf("1970-01-01"), 0, 1362653161);
assertBucketEquals("date", Date.valueOf("2015-11-19"), 16758, 8542395);
@ -309,6 +313,8 @@ public class TestHiveBucketing
return hiveValue;
case StandardTypes.VARCHAR:
return Slices.utf8Slice(hiveValue.toString());
case StandardTypes.CHAR:
return Slices.utf8Slice(hiveValue.toString());
case StandardTypes.DATE:
long daysSinceEpochInLocalZone = ((Date) hiveValue).toLocalDate().toEpochDay();
assertEquals(daysSinceEpochInLocalZone, DateWritable.dateToDays((Date) hiveValue));