jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix Alter Column DataType Issue

This commit is contained in:
SURYA SUMANTH N 2021-11-24 21:43:57 +05:30
parent cadaf36c32
commit e56166b183
8 changed files with 51 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveCoercionPolicy.java
View File

@ -56,13 +56,13 @@ public class HiveCoercionPolicy
return fromHiveType.equals(HiveType.HIVE_BYTE) || fromHiveType.equals(HiveType.HIVE_SHORT) || fromHiveType.equals(HiveType.HIVE_INT) || fromHiveType.equals(HiveType.HIVE_LONG);
}
if (fromHiveType.equals(HiveType.HIVE_BYTE)) {
return toHiveType.equals(HiveType.HIVE_SHORT) || toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG);
return toHiveType.equals(HiveType.HIVE_SHORT) || toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG) || toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE);
}
if (fromHiveType.equals(HiveType.HIVE_SHORT)) {
return toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG);
return toHiveType.equals(HiveType.HIVE_INT) || toHiveType.equals(HiveType.HIVE_LONG) || toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE);
}
if (fromHiveType.equals(HiveType.HIVE_INT)) {
return toHiveType.equals(HiveType.HIVE_LONG);
return toHiveType.equals(HiveType.HIVE_LONG) || toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE);
}
if (fromHiveType.equals(HiveType.HIVE_FLOAT)) {
return toHiveType.equals(HiveType.HIVE_DOUBLE) || toType instanceof DecimalType;
@ -70,7 +70,7 @@ public class HiveCoercionPolicy
if (fromHiveType.equals(HiveType.HIVE_DOUBLE)) {
return toHiveType.equals(HiveType.HIVE_FLOAT) || toType instanceof DecimalType;
}
if (fromType instanceof DecimalType) {
if (fromType instanceof DecimalType || fromHiveType.equals(HiveType.HIVE_LONG)) {
return toType instanceof DecimalType || toHiveType.equals(HiveType.HIVE_FLOAT) || toHiveType.equals(HiveType.HIVE_DOUBLE);
}

3
presto-orc/src/main/java/io/prestosql/orc/reader/AbstractNumericColumnReader.java
View File

@ -27,6 +27,7 @@ import io.prestosql.spi.type.DateType;
import io.prestosql.spi.type.IntegerType;
import io.prestosql.spi.type.SmallintType;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.VarcharType;
import org.openjdk.jol.info.ClassLayout;
import javax.annotation.Nullable;
@ -75,7 +76,7 @@ public abstract class AbstractNumericColumnReader<T>
throws OrcCorruptionException
{
requireNonNull(type, "type is null");
verifyStreamType(column, type, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType);
verifyStreamType(column, type, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType || t instanceof VarcharType);
this.column = requireNonNull(column, "column is null");
this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null");
}

3
presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReaders.java
View File

@ -19,6 +19,7 @@ import io.prestosql.orc.OrcBlockFactory.NestedBlockFactory;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.OrcRowDataCacheKey;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.type.Type;
@ -27,7 +28,7 @@ public final class ColumnReaders
private ColumnReaders() {}
public static ColumnReader createColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext, NestedBlockFactory blockFactory)
throws OrcCorruptionException
throws OrcCorruptionException, PrestoException
{
switch (column.getColumnType()) {
case BOOLEAN:

7
presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java
View File

@ -23,6 +23,7 @@ import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.stream.InputStreamSources;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.Type;
@ -40,6 +41,7 @@ import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DICTIO
import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT;
import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT_V2;
import static io.prestosql.orc.reader.ReaderUtils.verifyStreamType;
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.prestosql.spi.type.Chars.byteCountWithoutTrailingSpace;
import static io.prestosql.spi.type.Chars.isCharType;
import static io.prestosql.spi.type.VarbinaryType.isVarbinaryType;
@ -58,9 +60,12 @@ public class SliceColumnReader
private ColumnReader currentReader;
public SliceColumnReader(Type type, OrcColumn column, AggregatedMemoryContext systemMemoryContext)
throws OrcCorruptionException
throws OrcCorruptionException, PrestoException
{
requireNonNull(type, "type is null");
if (!(type instanceof VarcharType || type instanceof CharType || type instanceof VarbinaryType)) {
throw new PrestoException(NOT_SUPPORTED, String.format("Type %s can't be converted into type %s", type, column.getColumnType()));
}
verifyStreamType(column, type, t -> t instanceof VarcharType || t instanceof CharType || t instanceof VarbinaryType);
this.column = requireNonNull(column, "column is null");

31
presto-orc/src/test/java/io/prestosql/orc/TestAbstractNumbericColumnReader.java
View File

@ -17,15 +17,22 @@ import com.google.common.collect.ImmutableList;
import io.prestosql.memory.context.AggregatedMemoryContext;
import io.prestosql.orc.metadata.OrcColumnId;
import io.prestosql.orc.metadata.OrcType;
import io.prestosql.orc.reader.ColumnReader;
import io.prestosql.orc.reader.ColumnReaders;
import io.prestosql.orc.reader.DateColumnReader;
import io.prestosql.orc.reader.IntegerColumnReader;
import io.prestosql.orc.reader.LongColumnReader;
import io.prestosql.orc.reader.ShortColumnReader;
import io.prestosql.spi.type.VarcharType;
import org.testng.annotations.Test;
import static io.prestosql.testing.assertions.Assert.assertEquals;
public class TestAbstractNumbericColumnReader
{
private VarcharType type = VarcharType.VARCHAR;
@Test (expectedExceptions = OrcCorruptionException.class)
@Test
public void testTypeCoercionShort()
throws OrcCorruptionException
{
@ -36,10 +43,12 @@ public class TestAbstractNumbericColumnReader
OrcType.OrcTypeKind.SHORT,
new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"),
ImmutableList.of());
ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
ColumnReader actualShortColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
ShortColumnReader expectedShortColumnReader = new ShortColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
assertEquals(actualShortColumnReader.toString(), expectedShortColumnReader.toString());
}
@Test (expectedExceptions = OrcCorruptionException.class)
@Test
public void testTypeCoercionInteger()
throws OrcCorruptionException
{
@ -50,10 +59,12 @@ public class TestAbstractNumbericColumnReader
OrcType.OrcTypeKind.INT,
new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"),
ImmutableList.of());
ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
ColumnReader actualIntegerColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
IntegerColumnReader expectedIntegerColumnReader = new IntegerColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
assertEquals(actualIntegerColumnReader.toString(), expectedIntegerColumnReader.toString());
}
@Test (expectedExceptions = OrcCorruptionException.class)
@Test
public void testTypeCoercionBigInt()
throws OrcCorruptionException
{
@ -64,10 +75,12 @@ public class TestAbstractNumbericColumnReader
OrcType.OrcTypeKind.LONG,
new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"),
ImmutableList.of());
ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
ColumnReader actualLongColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
LongColumnReader expectedLongColumnReader = new LongColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
assertEquals(actualLongColumnReader.toString(), expectedLongColumnReader.toString());
}
@Test (expectedExceptions = OrcCorruptionException.class)
@Test
public void testTypeCoercionDate()
throws OrcCorruptionException
{
@ -78,6 +91,8 @@ public class TestAbstractNumbericColumnReader
OrcType.OrcTypeKind.DATE,
new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"),
ImmutableList.of());
ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
ColumnReader actualDateColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
DateColumnReader expectedDateColumnReader = new DateColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
assertEquals(actualDateColumnReader.toString(), expectedDateColumnReader.toString());
}
}

6
presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java
View File

@ -136,6 +136,12 @@ public class IntArrayBlock
return getInt(position, offset);
}
@Override
public String getString(int position, int offset, int length)
{
return String.valueOf(getInt(position, offset));
}
@Override
public boolean mayHaveNull()
{

4
presto-spi/src/main/java/io/prestosql/spi/type/AbstractType.java
View File

@ -114,13 +114,13 @@ public abstract class AbstractType
@Override
public Slice getSlice(Block block, int position)
{
throw new UnsupportedOperationException(getClass().getName());
throw new UnsupportedOperationException(String.format("Type Slice can't be converted to type %s", getClass().getName()));
}
@Override
public void writeSlice(BlockBuilder blockBuilder, Slice value)
{
throw new UnsupportedOperationException(getClass().getName());
throw new UnsupportedOperationException(String.format("Type Slice can't be converted to type %s", getClass().getName()));
}
@Override

6
presto-spi/src/main/java/io/prestosql/spi/type/VarcharType.java
View File

@ -188,6 +188,12 @@ public final class VarcharType
writeSlice(blockBuilder, Slices.utf8Slice(value));
}
@Override
public void writeLong(BlockBuilder blockBuilder, long value)
{
writeSlice(blockBuilder, Slices.utf8Slice(String.valueOf(value)));
}
@Override
public void writeSlice(BlockBuilder blockBuilder, Slice value)
{