From e56166b183dcb148df50c39b0c4ca0b16ce03466 Mon Sep 17 00:00:00 2001 From: SURYA SUMANTH N Date: Wed, 24 Nov 2021 21:43:57 +0530 Subject: [PATCH] Fix Alter Column DataType Issue --- .../plugin/hive/HiveCoercionPolicy.java | 8 ++--- .../reader/AbstractNumericColumnReader.java | 3 +- .../prestosql/orc/reader/ColumnReaders.java | 3 +- .../orc/reader/SliceColumnReader.java | 7 ++++- .../orc/TestAbstractNumbericColumnReader.java | 31 ++++++++++++++----- .../io/prestosql/spi/block/IntArrayBlock.java | 6 ++++ .../io/prestosql/spi/type/AbstractType.java | 4 +-- .../io/prestosql/spi/type/VarcharType.java | 6 ++++ 8 files changed, 51 insertions(+), 17 deletions(-) diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveCoercionPolicy.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveCoercionPolicy.java index 738cd6a0c..916887dd8 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveCoercionPolicy.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveCoercionPolicy.java @@ -56,13 +56,13 @@ public class HiveCoercionPolicy return fromHiveType.equals(HiveType.HIVE_BYTE) || fromHiveType.equals(HiveType.HIVE_SHORT) || fromHiveType.equals(HiveType.HIVE_INT) || fromHiveType.equals(HiveType.HIVE_LONG); } if (fromHiveType.equals(HiveType.HIVE_BYTE)) { - return toHiveType.equals(HiveType.HIVE_SHORT) || toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG); + return toHiveType.equals(HiveType.HIVE_SHORT) || toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG) || toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE); } if (fromHiveType.equals(HiveType.HIVE_SHORT)) { - return toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG); + return toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG) || toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE); } if (fromHiveType.equals(HiveType.HIVE_INT)) { - return toHiveType.equals(HiveType.HIVE_LONG); + return toHiveType.equals(HiveType.HIVE_LONG) || toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE); } if (fromHiveType.equals(HiveType.HIVE_FLOAT)) { return toHiveType.equals(HiveType.HIVE_DOUBLE) || toType instanceof DecimalType; @@ -70,7 +70,7 @@ public class HiveCoercionPolicy if (fromHiveType.equals(HiveType.HIVE_DOUBLE)) { return toHiveType.equals(HiveType.HIVE_FLOAT) || toType instanceof DecimalType; } - if (fromType instanceof DecimalType) { + if (fromType instanceof DecimalType || fromHiveType.equals(HiveType.HIVE_LONG)) { return toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE); } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractNumericColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractNumericColumnReader.java index 3e1a5b34d..e624d27c0 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractNumericColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractNumericColumnReader.java @@ -27,6 +27,7 @@ import io.prestosql.spi.type.DateType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.SmallintType; import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarcharType; import org.openjdk.jol.info.ClassLayout; import javax.annotation.Nullable; @@ -75,7 +76,7 @@ public abstract class AbstractNumericColumnReader throws OrcCorruptionException { requireNonNull(type, "type is null"); - verifyStreamType(column, type, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType); + verifyStreamType(column, type, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType || t instanceof VarcharType); this.column = requireNonNull(column, "column is null"); this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReaders.java b/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReaders.java index bd2d671ae..cabbf6d43 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReaders.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReaders.java @@ -19,6 +19,7 @@ import io.prestosql.orc.OrcBlockFactory.NestedBlockFactory; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; import io.prestosql.orc.OrcRowDataCacheKey; +import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; import io.prestosql.spi.type.Type; @@ -27,7 +28,7 @@ public final class ColumnReaders private ColumnReaders() {} public static ColumnReader createColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext, NestedBlockFactory blockFactory) - throws OrcCorruptionException + throws OrcCorruptionException, PrestoException { switch (column.getColumnType()) { case BOOLEAN: diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java index 131693ab9..b322be07b 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java @@ -23,6 +23,7 @@ import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind; import io.prestosql.orc.metadata.ColumnMetadata; import io.prestosql.orc.stream.InputStreamSources; +import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; import io.prestosql.spi.type.CharType; import io.prestosql.spi.type.Type; @@ -40,6 +41,7 @@ import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DICTIO import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT; import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT_V2; import static io.prestosql.orc.reader.ReaderUtils.verifyStreamType; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; import static io.prestosql.spi.type.Chars.byteCountWithoutTrailingSpace; import static io.prestosql.spi.type.Chars.isCharType; import static io.prestosql.spi.type.VarbinaryType.isVarbinaryType; @@ -58,9 +60,12 @@ public class SliceColumnReader private ColumnReader currentReader; public SliceColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext) - throws OrcCorruptionException + throws OrcCorruptionException, PrestoException { requireNonNull(type, "type is null"); + if (!(type instanceof VarcharType || type instanceof CharType || type instanceof VarbinaryType)) { + throw new PrestoException(NOT_SUPPORTED, String.format("Type %s can't be converted into type %s", type, column.getColumnType())); + } verifyStreamType(column, type, t -> t instanceof VarcharType || t instanceof CharType || t instanceof VarbinaryType); this.column = requireNonNull(column, "column is null"); diff --git a/presto-orc/src/test/java/io/prestosql/orc/TestAbstractNumbericColumnReader.java b/presto-orc/src/test/java/io/prestosql/orc/TestAbstractNumbericColumnReader.java index 734fb21e8..6ed48c88f 100644 --- a/presto-orc/src/test/java/io/prestosql/orc/TestAbstractNumbericColumnReader.java +++ b/presto-orc/src/test/java/io/prestosql/orc/TestAbstractNumbericColumnReader.java @@ -17,15 +17,22 @@ import com.google.common.collect.ImmutableList; import io.prestosql.memory.context.AggregatedMemoryContext; import io.prestosql.orc.metadata.OrcColumnId; import io.prestosql.orc.metadata.OrcType; +import io.prestosql.orc.reader.ColumnReader; import io.prestosql.orc.reader.ColumnReaders; +import io.prestosql.orc.reader.DateColumnReader; +import io.prestosql.orc.reader.IntegerColumnReader; +import io.prestosql.orc.reader.LongColumnReader; +import io.prestosql.orc.reader.ShortColumnReader; import io.prestosql.spi.type.VarcharType; import org.testng.annotations.Test; +import static io.prestosql.testing.assertions.Assert.assertEquals; + public class TestAbstractNumbericColumnReader { private VarcharType type = VarcharType.VARCHAR; - @Test (expectedExceptions = OrcCorruptionException.class) + @Test public void testTypeCoercionShort() throws OrcCorruptionException { @@ -36,10 +43,12 @@ public class TestAbstractNumbericColumnReader OrcType.OrcTypeKind.SHORT, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of()); - ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + ColumnReader actualShortColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + ShortColumnReader expectedShortColumnReader = new ShortColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + assertEquals(actualShortColumnReader.toString(), expectedShortColumnReader.toString()); } - @Test (expectedExceptions = OrcCorruptionException.class) + @Test public void testTypeCoercionInteger() throws OrcCorruptionException { @@ -50,10 +59,12 @@ public class TestAbstractNumbericColumnReader OrcType.OrcTypeKind.INT, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of()); - ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + ColumnReader actualIntegerColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + IntegerColumnReader expectedIntegerColumnReader = new IntegerColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + assertEquals(actualIntegerColumnReader.toString(), expectedIntegerColumnReader.toString()); } - @Test (expectedExceptions = OrcCorruptionException.class) + @Test public void testTypeCoercionBigInt() throws OrcCorruptionException { @@ -64,10 +75,12 @@ public class TestAbstractNumbericColumnReader OrcType.OrcTypeKind.LONG, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of()); - ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + ColumnReader actualLongColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + LongColumnReader expectedLongColumnReader = new LongColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + assertEquals(actualLongColumnReader.toString(), expectedLongColumnReader.toString()); } - @Test (expectedExceptions = OrcCorruptionException.class) + @Test public void testTypeCoercionDate() throws OrcCorruptionException { @@ -78,6 +91,8 @@ public class TestAbstractNumbericColumnReader OrcType.OrcTypeKind.DATE, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of()); - ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + ColumnReader actualDateColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null); + DateColumnReader expectedDateColumnReader = new DateColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName())); + assertEquals(actualDateColumnReader.toString(), expectedDateColumnReader.toString()); } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java index 752e82152..73d2421b2 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java @@ -136,6 +136,12 @@ public class IntArrayBlock return getInt(position, offset); } + @Override + public String getString(int position, int offset, int length) + { + return String.valueOf(getInt(position, offset)); + } + @Override public boolean mayHaveNull() { diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/AbstractType.java b/presto-spi/src/main/java/io/prestosql/spi/type/AbstractType.java index a4ccdea3d..0e580dad1 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/AbstractType.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/AbstractType.java @@ -114,13 +114,13 @@ public abstract class AbstractType @Override public Slice getSlice(Block block, int position) { - throw new UnsupportedOperationException(getClass().getName()); + throw new UnsupportedOperationException(String.format("Type Slice can't be converted to type %s", getClass().getName())); } @Override public void writeSlice(BlockBuilder blockBuilder, Slice value) { - throw new UnsupportedOperationException(getClass().getName()); + throw new UnsupportedOperationException(String.format("Type Slice can't be converted to type %s", getClass().getName())); } @Override diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/VarcharType.java b/presto-spi/src/main/java/io/prestosql/spi/type/VarcharType.java index de2e7d3b0..a6021f299 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/VarcharType.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/VarcharType.java @@ -188,6 +188,12 @@ public final class VarcharType writeSlice(blockBuilder, Slices.utf8Slice(value)); } + @Override + public void writeLong(BlockBuilder blockBuilder, long value) + { + writeSlice(blockBuilder, Slices.utf8Slice(String.valueOf(value))); + } + @Override public void writeSlice(BlockBuilder blockBuilder, Slice value) {