jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix for timestamp semantics

This commit is contained in:
NeerajUnnikrishnan 2021-06-07 11:21:38 -04:00
parent 4078ad7033
commit 1d6207faba
208 changed files with 3022 additions and 5399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java
View File

@ -18,7 +18,7 @@ import com.google.gson.Gson;
import io.prestosql.plugin.hive.HiveACIDWriteType;
import io.prestosql.plugin.hive.HiveFileWriter;
import io.prestosql.plugin.hive.HiveType;
import io.prestosql.plugin.hive.HiveWriteUtils;
import io.prestosql.plugin.hive.util.FieldSetterFactory;
import io.prestosql.spi.Page;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.Block;
@ -64,6 +64,7 @@ import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.log4j.Logger;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.UncheckedIOException;
@ -104,7 +105,7 @@ public class CarbondataFileWriter
private final Object row;
private final SettableStructObjectInspector tableInspector;
private final List<StructField> structFields;
private final HiveWriteUtils.FieldSetter[] setters;
private final FieldSetterFactory.FieldSetter[] setters;
private final Properties properties;
private final Optional<AcidOutputFormat.Options> acidOptions;
private final HiveACIDWriteType acidWriteType;
@ -183,9 +184,12 @@ public class CarbondataFileWriter
row = tableInspector.create();
setters = new HiveWriteUtils.FieldSetter[structFields.size()];
setters = new FieldSetterFactory.FieldSetter[structFields.size()];
FieldSetterFactory fieldSetterFactory = new FieldSetterFactory(DateTimeZone.UTC);
for (int i = 0; i < setters.length; i++) {
setters[i] = HiveWriteUtils.createFieldSetter(tableInspector, row, structFields.get(i),
setters[i] = fieldSetterFactory.create(tableInspector, row, structFields.get(i),
fileColumnTypes.get(structFields.get(i).getFieldID()));
}

9
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java
View File

@ -142,7 +142,6 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.log4j.Logger;
import org.joda.time.DateTimeZone;
import java.io.File;
import java.io.IOException;
@ -248,8 +247,8 @@ public class CarbondataMetadata
}
public CarbondataMetadata(SemiTransactionalHiveMetastore metastore,
HdfsEnvironment hdfsEnvironment, HivePartitionManager partitionManager, DateTimeZone timeZone,
boolean allowCorruptWritesForTesting, boolean writesToNonManagedTablesEnabled,
HdfsEnvironment hdfsEnvironment, HivePartitionManager partitionManager,
boolean writesToNonManagedTablesEnabled,
boolean createsOfNonManagedTablesEnabled, boolean tableCreatesWithLocationAllowed,
TypeManager typeManager, LocationService locationService,
JsonCodec<PartitionUpdate> partitionUpdateCodec,
@ -259,7 +258,7 @@ public class CarbondataMetadata
CarbondataTableReader carbondataTableReader, String carbondataTableStore, long carbondataMajorVacuumSegSize, long carbondataMinorVacuumSegCount,
ScheduledExecutorService executorService, ScheduledExecutorService hiveMetastoreClientService)
{
super(metastore, hdfsEnvironment, partitionManager, timeZone, allowCorruptWritesForTesting,
super(metastore, hdfsEnvironment, partitionManager,
writesToNonManagedTablesEnabled, createsOfNonManagedTablesEnabled, tableCreatesWithLocationAllowed,
typeManager, locationService, partitionUpdateCodec, typeTranslator, hetuVersion,
hiveStatisticsProvider, accessControlMetadata, false, 2, 0.0, executorService,
@ -1145,8 +1144,6 @@ public class CarbondataMetadata
@Override
public CarbondataOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
{
verifyJvmTimeZone();
// get the root directory for the database
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();

23
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java
View File

@ -37,7 +37,6 @@ import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
import io.prestosql.plugin.hive.security.AccessControlMetadataFactory;
import io.prestosql.plugin.hive.statistics.MetastoreHiveStatisticsProvider;
import io.prestosql.spi.type.TypeManager;
import org.joda.time.DateTimeZone;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
@ -50,7 +49,6 @@ public class CarbondataMetadataFactory
extends HiveMetadataFactory
{
private static final Logger log = Logger.get(HiveMetadataFactory.class);
private final boolean allowCorruptWritesForTesting;
private final boolean skipDeletionForAlter;
private final boolean skipTargetCleanupOnRollback;
private final boolean writesToNonManagedTablesEnabled;
@ -60,7 +58,6 @@ public class CarbondataMetadataFactory
private final HiveMetastore metastore;
private final HdfsEnvironment hdfsEnvironment;
private final HivePartitionManager partitionManager;
private final DateTimeZone timeZone;
private final TypeManager typeManager;
private final LocationService locationService;
private final BoundedExecutor renameExecution;
@ -93,9 +90,8 @@ public class CarbondataMetadataFactory
AccessControlMetadataFactory accessControlMetadataFactory,
CarbondataTableReader carbondataTableReader)
{
this(metastore, hdfsEnvironment, partitionManager, carbondataConfig.getDateTimeZone(),
this(metastore, hdfsEnvironment, partitionManager,
carbondataConfig.getMaxConcurrentFileRenames(),
carbondataConfig.getAllowCorruptWritesForTesting(),
carbondataConfig.isSkipDeletionForAlter(),
carbondataConfig.isSkipTargetCleanupOnRollback(),
true,
@ -112,9 +108,9 @@ public class CarbondataMetadataFactory
}
public CarbondataMetadataFactory(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment,
HivePartitionManager partitionManager, DateTimeZone timeZone,
HivePartitionManager partitionManager,
int maxConcurrentFileRenames,
boolean allowCorruptWritesForTesting, boolean skipDeletionForAlter,
boolean skipDeletionForAlter,
boolean skipTargetCleanupOnRollback, boolean writesToNonManagedTablesEnabled,
boolean createsOfNonManagedTablesEnabled, boolean tableCreatesWithLocationAllowed,
long perTransactionCacheMaximumSize,
@ -133,9 +129,7 @@ public class CarbondataMetadataFactory
super(metastore,
hdfsEnvironment,
partitionManager,
timeZone,
maxConcurrentFileRenames,
allowCorruptWritesForTesting,
skipDeletionForAlter,
skipTargetCleanupOnRollback,
writesToNonManagedTablesEnabled,
@ -157,7 +151,6 @@ public class CarbondataMetadataFactory
2, 0.0, false,
Optional.of(new Duration(5, TimeUnit.MINUTES)),
hmsWriteBatchSize);
this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
this.skipDeletionForAlter = skipDeletionForAlter;
this.skipTargetCleanupOnRollback = skipTargetCleanupOnRollback;
this.writesToNonManagedTablesEnabled = writesToNonManagedTablesEnabled;
@ -167,7 +160,6 @@ public class CarbondataMetadataFactory
this.metastore = requireNonNull(metastore, "metastore is null");
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
this.timeZone = requireNonNull(timeZone, "timeZone is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.locationService = requireNonNull(locationService, "locationService is null");
this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null");
@ -176,13 +168,6 @@ public class CarbondataMetadataFactory
this.hetuVersion = requireNonNull(hetuVersion, "hetuVersion is null");
this.accessControlMetadataFactory = requireNonNull(accessControlMetadataFactory,
"accessControlMetadataFactory is null");
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
log.warn(
"Hive writes are disabled. To write data to Hive, your JVM timezone must match the " +
"Hive storage timezone. Add -Duser.timezone=%s to your JVM arguments",
timeZone.getID());
}
this.renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
this.vacuumExecutorService = requireNonNull(vacuumExecutorService, "vacuumExecutorService is null");
this.hiveMetastoreClientService = requireNonNull(hiveMetastoreClientService, "hiveMetastoreClientService is null");
@ -218,8 +203,6 @@ public class CarbondataMetadataFactory
return new CarbondataMetadata(metastore,
this.hdfsEnvironment,
this.partitionManager,
this.timeZone,
this.allowCorruptWritesForTesting,
this.writesToNonManagedTablesEnabled,
this.createsOfNonManagedTablesEnabled,
this.tableCreatesWithLocationAllowed,

3
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java
View File

@ -47,6 +47,7 @@ import java.util.Set;
import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION;
import static org.joda.time.DateTimeZone.UTC;
public class CarbondataWriterFactory
extends HiveWriterFactory
@ -87,7 +88,7 @@ public class CarbondataWriterFactory
additionalTableParameters, bucketCount, sortedBy, locationHandle,
locationService, queryId, pageSinkMetadataProvider,
typeManager, hdfsEnvironment, pageSorter, sortBufferSize,
maxOpenSortFiles, immutablePartitions, session, nodeManager,
maxOpenSortFiles, immutablePartitions, UTC, session, nodeManager,
eventClient, hiveSessionProperties, hiveWriterStats, orcFileWriterFactory);
this.additionalJobConf = requireNonNull(additionalJobConf, "Additional JobConf is null");

6
hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterMetadata.java
View File

@ -104,12 +104,6 @@ public class TestDataCenterMetadata
return 0;
}
@Override
public boolean isLegacyTimestamp()
{
return false;
}
@Override
public <T> T getProperty(String name, Class<T> type)
{

4
hetu-greenplum/src/main/java/io/hetu/core/plugin/greenplum/GreenPlumSqlClient.java
View File

@ -412,7 +412,7 @@ public class GreenPlumSqlClient
return Optional.of(typedVarcharColumnMapping(jdbcTypeName));
}
if (typeHandle.getJdbcType() == Types.TIMESTAMP) {
return Optional.of(timestampColumnMapping(session));
return Optional.of(timestampColumnMapping());
}
if (typeHandle.getJdbcType() == Types.ARRAY && supportArrays) {
if (!typeHandle.getArrayDimensions().isPresent()) {
@ -447,7 +447,7 @@ public class GreenPlumSqlClient
return WriteMapping.sliceMapping("bytea", varbinaryWriteFunction());
}
if (TIMESTAMP.equals(type)) {
return WriteMapping.longMapping("timestamp", timestampWriteFunction(session));
return WriteMapping.longMapping("timestamp", timestampWriteFunction());
}
if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
return WriteMapping.longMapping("timestamp with time zone", timestampWithTimeZoneWriteFunction());

26
hetu-greenplum/src/test/java/io/hetu/core/plugin/greenplum/TestGreenPlumTypeMapping.java
View File

@ -522,7 +522,7 @@ public class TestGreenPlumTypeMapping
}
@Test(dataProvider = "testTimestampDataProvider")
public void testTimestamp(boolean legacyTimestamp, boolean insertWithPresto)
public void testTimestamp(boolean insertWithPresto)
{
// using two non-JVM zones so that we don't need to worry what Postgres system zone is
for (ZoneId sessionZone : ImmutableList.of(ZoneOffset.UTC, jvmZone, vilnius, kathmandu, ZoneId.of(TestingSession.DEFAULT_TIME_ZONE_KEY.getId()))) {
@ -534,17 +534,16 @@ public class TestGreenPlumTypeMapping
if (!insertWithPresto) {
// when writing, Postgres JDBC driver converts LocalDateTime to string representing date-time in JVM zone
addTimestampTestIfSupported(tests, legacyTimestamp, sessionZone, epoch); // epoch also is a gap in JVM zone
addTimestampTestIfSupported(tests, legacyTimestamp, sessionZone, timeGapInJvmZone1);
addTimestampTestIfSupported(tests, legacyTimestamp, sessionZone, timeGapInJvmZone2);
addTimestampTestIfSupported(tests, epoch); // epoch also is a gap in JVM zone
addTimestampTestIfSupported(tests, timeGapInJvmZone1);
addTimestampTestIfSupported(tests, timeGapInJvmZone2);
}
addTimestampTestIfSupported(tests, legacyTimestamp, sessionZone, timeGapInVilnius);
addTimestampTestIfSupported(tests, legacyTimestamp, sessionZone, timeGapInKathmandu);
addTimestampTestIfSupported(tests, timeGapInVilnius);
addTimestampTestIfSupported(tests, timeGapInKathmandu);
Session session = Session.builder(getQueryRunner().getDefaultSession())
.setTimeZoneKey(TimeZoneKey.getTimeZoneKey(sessionZone.getId()))
.setSystemProperty("legacy_timestamp", Boolean.toString(legacyTimestamp))
.build();
if (insertWithPresto) {
@ -556,13 +555,8 @@ public class TestGreenPlumTypeMapping
}
}
private void addTimestampTestIfSupported(DataTypeTest tests, boolean legacyTimestamp, ZoneId sessionZone, LocalDateTime dateTime)
private void addTimestampTestIfSupported(DataTypeTest tests, LocalDateTime dateTime)
{
if (legacyTimestamp && isGap(sessionZone, dateTime)) {
// in legacy timestamp semantics we cannot represent this dateTime
return;
}
tests.addRoundTrip(timestampDataType(), dateTime);
}
@ -570,10 +564,8 @@ public class TestGreenPlumTypeMapping
public Object[][] testTimestampDataProvider()
{
return new Object[][] {
{true, true},
{false, true},
{true, false},
{false, false},
{true},
{false},
};
}

6
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/client/TestingConnectorSession.java
View File

@ -87,12 +87,6 @@ public class TestingConnectorSession
return 0;
}
@Override
public boolean isLegacyTimestamp()
{
return false;
}
@Override
public <T> T getProperty(String name, Class<T> type)
{

4
hetu-opengauss/src/main/java/io/hetu/core/plugin/opengauss/OpenGaussClient.java
View File

@ -70,7 +70,7 @@ public class OpenGaussClient
return WriteMapping.sliceMapping("bytea", varbinaryWriteFunction());
}
if (TIMESTAMP.equals(type)) {
return WriteMapping.longMapping("timestamp", timestampWriteFunctionUsingSqlTimestamp(session));
return WriteMapping.longMapping("timestamp", timestampWriteFunctionUsingSqlTimestamp());
}
if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
return WriteMapping.longMapping("timestamp with time zone", timestampWithTimeZoneWriteFunction());
@ -140,7 +140,7 @@ public class OpenGaussClient
return Optional.of(typedVarcharColumnMapping(jdbcTypeName));
}
if (typeHandle.getJdbcType() == Types.TIMESTAMP) {
return Optional.of(timestampColumnMappingUsingSqlTimestamp(session));
return Optional.of(timestampColumnMappingUsingSqlTimestamp());
}
if (typeHandle.getJdbcType() == Types.ARRAY && supportArrays) {
if (!typeHandle.getArrayDimensions().isPresent()) {

34
hetu-oracle/src/main/java/io/hetu/core/plugin/oracle/OracleClient.java
View File

@ -73,7 +73,6 @@ import java.sql.Timestamp;
import java.sql.Types;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -271,7 +270,6 @@ public class OracleClient
/**
* timestamp with time zone
*
* @param connectorSession connectorSession
* @return LongWriteFunction
* @deprecated This method uses {@link java.sql.Timestamp} and the class cannot
* represent date-time value when JVM zone had
@ -281,28 +279,11 @@ public class OracleClient
* supports {@link LocalDateTime}, use
*/
@Deprecated
public static LongWriteFunction timestampWithTimeZoneWriteFunctionUsingSqlTimestamp(
ConnectorSession connectorSession)
public static LongWriteFunction timestampWithTimeZoneWriteFunctionUsingSqlTimestamp()
{
if (connectorSession.isLegacyTimestamp()) {
ZoneId sessionZone = ZoneId.of(connectorSession.getTimeZoneKey().getId());
return (statement, index, value) -> setTimestampWithTimeZoneLegacy(statement, index, value, sessionZone);
}
return (statement, index, value) -> setTimestampWithTimeZone(statement, index, value);
}
private static void setTimestampWithTimeZoneLegacy(PreparedStatement statement, int index, long value,
ZoneId sessionZone)
{
try {
statement.setTimestamp(index, new Timestamp(DateTimeEncoding.unpackMillisUtc(
fromHetuLegacyTimestamp(value, sessionZone).atZone(sessionZone).toInstant().toEpochMilli())));
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, "Hetu Oracle connector failed to set Timestamp With Time Zone Legacy");
}
}
private static void setTimestampWithTimeZone(PreparedStatement statement, int index, long value)
{
try {
@ -314,11 +295,6 @@ public class OracleClient
}
}
private static LocalDateTime fromHetuLegacyTimestamp(long value, ZoneId sessionZone)
{
return Instant.ofEpochMilli(value).atZone(sessionZone).toLocalDateTime();
}
private static LocalDateTime fromHetuTimestamp(long value)
{
return Instant.ofEpochMilli(value).atZone(UTC).toLocalDateTime();
@ -558,7 +534,7 @@ public class OracleClient
break;
case OracleTypes.TIMESTAMP:
columnMapping = Optional.of(timestampColumnMappingUsingSqlTimestamp(session));
columnMapping = Optional.of(timestampColumnMappingUsingSqlTimestamp());
break;
// the following two data type is not supported because of oracle.sql.TIMESTAMPTZ
@ -680,14 +656,14 @@ public class OracleClient
return WriteMapping.sliceMapping("BLOB", varbinaryWriteFunction());
}
else if (TIMESTAMP.equals(type)) {
return WriteMapping.longMapping("TIMESTAMP", timestampWriteFunctionUsingSqlTimestamp(session));
return WriteMapping.longMapping("TIMESTAMP", timestampWriteFunctionUsingSqlTimestamp());
}
else if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
return WriteMapping.longMapping("TIMESTAMP(3) WITH TIME ZONE",
timestampWithTimeZoneWriteFunctionUsingSqlTimestamp(session));
timestampWithTimeZoneWriteFunctionUsingSqlTimestamp());
}
else if (DATE.equals(type)) {
return WriteMapping.longMapping("DATE", timestampWriteFunctionUsingSqlTimestamp(session));
return WriteMapping.longMapping("DATE", timestampWriteFunctionUsingSqlTimestamp());
}
else {
throw new PrestoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName());

2
pom.xml
View File

@ -514,7 +514,7 @@
<dependency>
<groupId>io.prestosql.hive</groupId>
<artifactId>hive-apache</artifactId>
<version>3.0.0-2</version>
<version>3.1.2-1</version>
</dependency>
<dependency>

2
presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcClient.java
View File

@ -316,7 +316,7 @@ public class BaseJdbcClient
@Override
public Optional<ColumnMapping> toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle)
{
return jdbcTypeToPrestoType(session, typeHandle);
return jdbcTypeToPrestoType(typeHandle);
}
@Override

67
presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/StandardColumnMappings.java
View File

@ -16,7 +16,6 @@ package io.prestosql.plugin.jdbc;
import com.google.common.base.CharMatcher;
import com.google.common.primitives.Shorts;
import com.google.common.primitives.SignedBytes;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.Decimals;
@ -35,7 +34,6 @@ import java.sql.Timestamp;
import java.sql.Types;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkArgument;
@ -287,42 +285,23 @@ public final class StandardColumnMappings
* {@link #timestampColumnMapping} instead.
*/
@Deprecated
public static ColumnMapping timestampColumnMappingUsingSqlTimestamp(ConnectorSession session)
public static ColumnMapping timestampColumnMappingUsingSqlTimestamp()
{
if (session.isLegacyTimestamp()) {
ZoneId sessionZone = ZoneId.of(session.getTimeZoneKey().getId());
return ColumnMapping.longMapping(
TIMESTAMP,
(resultSet, columnIndex) -> {
Timestamp timestamp = resultSet.getTimestamp(columnIndex);
return toPrestoLegacyTimestamp(timestamp.toLocalDateTime(), sessionZone);
},
timestampWriteFunctionUsingSqlTimestamp(session));
}
return ColumnMapping.longMapping(
TIMESTAMP,
(resultSet, columnIndex) -> {
Timestamp timestamp = resultSet.getTimestamp(columnIndex);
return toPrestoTimestamp(timestamp.toLocalDateTime());
},
timestampWriteFunctionUsingSqlTimestamp(session));
timestampWriteFunctionUsingSqlTimestamp());
}
public static ColumnMapping timestampColumnMapping(ConnectorSession session)
public static ColumnMapping timestampColumnMapping()
{
if (session.isLegacyTimestamp()) {
ZoneId sessionZone = ZoneId.of(session.getTimeZoneKey().getId());
return ColumnMapping.longMapping(
TIMESTAMP,
(resultSet, columnIndex) -> toPrestoLegacyTimestamp(resultSet.getObject(columnIndex, LocalDateTime.class), sessionZone),
timestampWriteFunction(session));
}
return ColumnMapping.longMapping(
TIMESTAMP,
(resultSet, columnIndex) -> toPrestoTimestamp(resultSet.getObject(columnIndex, LocalDateTime.class)),
timestampWriteFunction(session));
timestampWriteFunction());
}
/**
@ -332,33 +311,14 @@ public final class StandardColumnMappings
* {@link #timestampWriteFunction} instead.
*/
@Deprecated
public static LongWriteFunction timestampWriteFunctionUsingSqlTimestamp(ConnectorSession connectorSession)
public static LongWriteFunction timestampWriteFunctionUsingSqlTimestamp()
{
if (connectorSession.isLegacyTimestamp()) {
ZoneId sessionZone = ZoneId.of(connectorSession.getTimeZoneKey().getId());
return (statement, index, value) -> statement.setTimestamp(index, Timestamp.valueOf(fromPrestoLegacyTimestamp(value, sessionZone)));
}
return (statement, index, value) -> statement.setTimestamp(index, Timestamp.valueOf(fromPrestoTimestamp(value)));
}
public static LongWriteFunction timestampWriteFunction(ConnectorSession session)
public static LongWriteFunction timestampWriteFunction()
{
if (session.isLegacyTimestamp()) {
ZoneId sessionZone = ZoneId.of(session.getTimeZoneKey().getId());
return (statement, index, value) -> statement.setObject(index, fromPrestoLegacyTimestamp(value, sessionZone));
}
return (statement, index, value) -> {
statement.setObject(index, fromPrestoTimestamp(value));
};
}
/**
* @deprecated applicable in legacy timestamp semantics only
*/
@Deprecated
private static long toPrestoLegacyTimestamp(LocalDateTime localDateTime, ZoneId sessionZone)
{
return localDateTime.atZone(sessionZone).toInstant().toEpochMilli();
return (statement, index, value) -> statement.setObject(index, fromPrestoTimestamp(value));
}
private static long toPrestoTimestamp(LocalDateTime localDateTime)
@ -366,21 +326,12 @@ public final class StandardColumnMappings
return localDateTime.atZone(UTC).toInstant().toEpochMilli();
}
/**
* @deprecated applicable in legacy timestamp semantics only
*/
@Deprecated
private static LocalDateTime fromPrestoLegacyTimestamp(long value, ZoneId sessionZone)
{
return Instant.ofEpochMilli(value).atZone(sessionZone).toLocalDateTime();
}
private static LocalDateTime fromPrestoTimestamp(long value)
{
return Instant.ofEpochMilli(value).atZone(UTC).toLocalDateTime();
}
public static Optional<ColumnMapping> jdbcTypeToPrestoType(ConnectorSession session, JdbcTypeHandle type)
public static Optional<ColumnMapping> jdbcTypeToPrestoType(JdbcTypeHandle type)
{
int columnSize = type.getColumnSize();
switch (type.getJdbcType()) {
@ -444,7 +395,7 @@ public final class StandardColumnMappings
case Types.TIMESTAMP:
// TODO default to `timestampColumnMapping`
return Optional.of(timestampColumnMappingUsingSqlTimestamp(session));
return Optional.of(timestampColumnMappingUsingSqlTimestamp());
}
return Optional.empty();
}

11
presto-base-jdbc/src/test/java/io/prestosql/plugin/jdbc/TestJdbcQueryBuilder.java
View File

@ -22,8 +22,6 @@ import io.prestosql.spi.predicate.Range;
import io.prestosql.spi.predicate.SortedRangeSet;
import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.SqlTimestamp;
import io.prestosql.testing.DateTimeTestingUtils;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@ -64,16 +62,15 @@ import static io.prestosql.spi.type.IntegerType.INTEGER;
import static io.prestosql.spi.type.RealType.REAL;
import static io.prestosql.spi.type.SmallintType.SMALLINT;
import static io.prestosql.spi.type.TimeType.TIME;
import static io.prestosql.spi.type.TimeZoneKey.UTC_KEY;
import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
import static io.prestosql.spi.type.TinyintType.TINYINT;
import static io.prestosql.spi.type.VarcharType.VARCHAR;
import static io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf;
import static io.prestosql.testing.TestingConnectorSession.SESSION;
import static java.lang.Float.floatToRawIntBits;
import static java.lang.String.format;
import static java.time.temporal.ChronoUnit.DAYS;
import static java.util.function.Function.identity;
import static org.joda.time.DateTimeZone.UTC;
import static org.testng.Assert.assertEquals;
@Test(singleThreaded = true)
@ -398,11 +395,7 @@ public class TestJdbcQueryBuilder
private static long toPrestoTimestamp(int year, int month, int day, int hour, int minute, int second)
{
SqlTimestamp sqlTimestamp = DateTimeTestingUtils.sqlTimestampOf(year, month, day, hour, minute, second, 0, UTC, UTC_KEY, SESSION);
if (SESSION.isLegacyTimestamp()) {
return sqlTimestamp.getMillisUtc();
}
return sqlTimestamp.getMillis();
return sqlTimestampOf(year, month, day, hour, minute, second, 0).getMillis();
}
private static Timestamp toTimestamp(int year, int month, int day, int hour, int minute, int second)

16
presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPageSource.java
View File

@ -38,7 +38,6 @@ import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.BlockBuilder;
import io.prestosql.spi.block.PageBuilderStatus;
import io.prestosql.spi.connector.ConnectorPageSource;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.type.ArrayType;
import io.prestosql.spi.type.RowType;
import io.prestosql.spi.type.Type;
@ -89,7 +88,6 @@ public class ElasticsearchPageSource
public ElasticsearchPageSource(
ElasticsearchClient client,
ConnectorSession session,
ElasticsearchTableHandle table,
ElasticsearchSplit split,
List<ElasticsearchColumnHandle> columns)
@ -99,7 +97,7 @@ public class ElasticsearchPageSource
this.columns = ImmutableList.copyOf(columns);
decoders = createDecoders(session, columns);
decoders = createDecoders(columns);
// When the _source field is requested, we need to bypass column pruning when fetching the document
boolean needAllFields = columns.stream()
@ -252,7 +250,7 @@ public class ElasticsearchPageSource
}
}
private List<Decoder> createDecoders(ConnectorSession session, List<ElasticsearchColumnHandle> columns)
private List<Decoder> createDecoders(List<ElasticsearchColumnHandle> columns)
{
return columns.stream()
.map(column -> {
@ -268,12 +266,12 @@ public class ElasticsearchPageSource
return new SourceColumnDecoder();
}
return createDecoder(session, column.getName(), column.getType());
return createDecoder(column.getName(), column.getType());
})
.collect(toImmutableList());
}
private Decoder createDecoder(ConnectorSession session, String path, Type type)
private Decoder createDecoder(String path, Type type)
{
if (type.equals(VARCHAR)) {
return new VarcharDecoder(path);
@ -282,7 +280,7 @@ public class ElasticsearchPageSource
return new VarbinaryDecoder(path);
}
else if (type.equals(TIMESTAMP)) {
return new TimestampDecoder(session, path);
return new TimestampDecoder(path);
}
else if (type.equals(BOOLEAN)) {
return new BooleanDecoder(path);
@ -312,7 +310,7 @@ public class ElasticsearchPageSource
RowType rowType = (RowType) type;
List<Decoder> decoders = rowType.getFields().stream()
.map(field -> createDecoder(session, appendPath(path, field.getName().get()), field.getType()))
.map(field -> createDecoder(appendPath(path, field.getName().get()), field.getType()))
.collect(toImmutableList());
List<String> fieldNames = rowType.getFields().stream()
@ -324,7 +322,7 @@ public class ElasticsearchPageSource
}
if (type instanceof ArrayType) {
Type elementType = ((ArrayType) type).getElementType();
return new ArrayDecoder(path, createDecoder(session, path, elementType));
return new ArrayDecoder(path, createDecoder(path, elementType));
}
throw new UnsupportedOperationException("Type not supported: " + type);

1
presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPageSourceProvider.java
View File

@ -47,7 +47,6 @@ public class ElasticsearchPageSourceProvider
return new ElasticsearchPageSource(
client,
session,
(ElasticsearchTableHandle) table, (ElasticsearchSplit) split,
columns.stream()
.map(ElasticsearchColumnHandle.class::cast)

9
presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/decoders/TimestampDecoder.java
View File

@ -15,12 +15,11 @@ package io.prestosql.elasticsearch.decoders;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.BlockBuilder;
import io.prestosql.spi.connector.ConnectorSession;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.search.SearchHit;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.function.Supplier;
import static io.prestosql.spi.StandardErrorCode.TYPE_MISMATCH;
@ -32,12 +31,10 @@ public class TimestampDecoder
implements Decoder
{
private final String path;
private final ZoneId zoneId;
public TimestampDecoder(ConnectorSession session, String path)
public TimestampDecoder(String path)
{
this.path = path;
this.zoneId = ZoneId.of(session.getTimeZoneKey().getId());
}
@Override
@ -53,7 +50,7 @@ public class TimestampDecoder
else {
TIMESTAMP.writeLong(output,
ISO_DATE_TIME.parse(documentField.getValue(), LocalDateTime::from)
.atZone(zoneId)
.atOffset(ZoneOffset.UTC)
.toInstant()
.toEpochMilli());
}

35
presto-hive/src/main/java/io/prestosql/plugin/hive/GenericHiveRecordCursor.java
View File

@ -26,7 +26,9 @@ import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
@ -40,17 +42,13 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.RecordReader;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
@ -105,7 +103,6 @@ class GenericHiveRecordCursor<K, V extends Writable>
private final boolean[] nulls;
private final long totalBytes;
private final DateTimeZone hiveStorageTimeZone;
private long completedBytes;
private Object rowData;
@ -118,7 +115,6 @@ class GenericHiveRecordCursor<K, V extends Writable>
long totalBytes,
Properties splitSchema,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager)
{
requireNonNull(path, "path is null");
@ -126,14 +122,12 @@ class GenericHiveRecordCursor<K, V extends Writable>
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(columns, "columns is null");
requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.path = path;
this.recordReader = recordReader;
this.totalBytes = totalBytes;
this.key = recordReader.createKey();
this.value = recordReader.createValue();
this.hiveStorageTimeZone = hiveStorageTimeZone;
this.deserializer = getDeserializer(configuration, splitSchema);
this.rowInspector = getTableObjectInspector(deserializer);
@ -278,35 +272,18 @@ class GenericHiveRecordCursor<K, V extends Writable>
else {
Object fieldValue = ((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
checkState(fieldValue != null, "fieldValue should not be null");
longs[column] = getLongExpressedValue(fieldValue, hiveStorageTimeZone);
longs[column] = getLongExpressedValue(fieldValue);
nulls[column] = false;
}
}
private static long getLongExpressedValue(Object value, DateTimeZone hiveTimeZone)
private long getLongExpressedValue(Object value)
{
if (value instanceof Date) {
long storageTime = ((Date) value).getTime();
// convert date from VM current time zone to UTC
long utcMillis = storageTime + DateTimeZone.getDefault().getOffset(storageTime);
return TimeUnit.MILLISECONDS.toDays(utcMillis);
return ((Date) value).toEpochDay();
}
if (value instanceof Timestamp) {
// The Hive SerDe parses timestamps using the default time zone of
// this JVM, but the data might have been written using a different
// time zone. We need to convert it to the configured time zone.
// the timestamp that Hive parsed using the JVM time zone
long parsedJvmMillis = ((Timestamp) value).getTime();
// remove the JVM time zone correction from the timestamp
DateTimeZone jvmTimeZone = DateTimeZone.getDefault();
long hiveMillis = jvmTimeZone.convertUTCToLocal(parsedJvmMillis);
// convert to UTC using the real time zone for the underlying data
long utcMillis = hiveTimeZone.convertLocalToUTC(hiveMillis, false);
return utcMillis;
return ((Timestamp) value).toEpochMilli();
}
if (value instanceof Float) {
return floatToRawIntBits(((Float) value));

3
presto-hive/src/main/java/io/prestosql/plugin/hive/GenericHiveRecordCursorProvider.java
View File

@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.RecordReader;
import org.joda.time.DateTimeZone;
import javax.inject.Inject;
@ -57,7 +56,6 @@ public class GenericHiveRecordCursorProvider
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
boolean s3SelectPushdownEnabled,
Map<String, String> customSplitInfo)
@ -80,7 +78,6 @@ public class GenericHiveRecordCursorProvider
length,
schema,
columns,
hiveStorageTimeZone,
typeManager));
});
}

22
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveBucketing.java
View File

@ -170,7 +170,7 @@ public final class HiveBucketing
return Optional.empty();
}
if (containsTimestampBucketedV2(table.getStorage().getBucketProperty().get(), table)) {
if (bucketedOnTimestamp(table.getStorage().getBucketProperty().get(), table)) {
return Optional.empty();
}
@ -269,35 +269,27 @@ public final class HiveBucketing
}
}
public static boolean containsTimestampBucketedV2(HiveBucketProperty bucketProperty, Table table)
public static boolean bucketedOnTimestamp(HiveBucketProperty bucketProperty, Table table)
{
switch (bucketProperty.getBucketingVersion()) {
case BUCKETING_V1:
return false;
case BUCKETING_V2:
break;
default:
throw new IllegalArgumentException("Unsupported bucketing version: " + bucketProperty.getBucketingVersion());
}
return bucketProperty.getBucketedBy().stream()
.map(columnName -> table.getColumn(columnName)
.orElseThrow(() -> new IllegalArgumentException(format("Cannot find column '%s' in %s", columnName, table))))
.map(Column::getType)
.map(HiveType::getTypeInfo)
.anyMatch(HiveBucketing::containsTimestampBucketedV2);
.anyMatch(HiveBucketing::bucketedOnTimestamp);
}
private static boolean containsTimestampBucketedV2(TypeInfo type)
private static boolean bucketedOnTimestamp(TypeInfo type)
{
switch (type.getCategory()) {
case PRIMITIVE:
return ((PrimitiveTypeInfo) type).getPrimitiveCategory() == TIMESTAMP;
case LIST:
return containsTimestampBucketedV2(((ListTypeInfo) type).getListElementTypeInfo());
return bucketedOnTimestamp(((ListTypeInfo) type).getListElementTypeInfo());
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) type;
// Note: we do not check map value type because HiveBucketingV2#hashOfMap hashes map values with v1
return containsTimestampBucketedV2(mapTypeInfo.getMapKeyTypeInfo());
return bucketedOnTimestamp(mapTypeInfo.getMapKeyTypeInfo()) ||
bucketedOnTimestamp(mapTypeInfo.getMapValueTypeInfo());
default:
// TODO: support more types, e.g. ROW
throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive category: " + type.getCategory());

82
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java
View File

@ -57,13 +57,12 @@ import static java.util.concurrent.TimeUnit.MINUTES;
"hive.optimized-reader.enabled",
"hive.orc.optimized-writer.enabled",
"hive.rcfile-optimized-writer.enabled",
"hive.time-zone",
})
public class HiveConfig
{
private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
private String timeZone = TimeZone.getDefault().getID();
private DataSize maxSplitSize = new DataSize(64, MEGABYTE);
private int maxPartitionsPerScan = 100_000;
private int maxOutstandingSplits = 1_000;
@ -115,6 +114,9 @@ public class HiveConfig
private DataSize textMaxLineLength = new DataSize(100, MEGABYTE);
private String orcLegacyTimeZone = TimeZone.getDefault().getID();
private String parquetTimeZone = TimeZone.getDefault().getID();
private boolean useParquetColumnNames;
private boolean failOnCorruptedParquetStatistics = true;
private DataSize parquetMaxReadBlockSize = new DataSize(16, MEGABYTE);
@ -150,6 +152,7 @@ public class HiveConfig
private Duration orcRowDataCacheTtl = new Duration(4, HOURS);
private DataSize orcRowDataCacheMaximumWeight = new DataSize(20, GIGABYTE);
private String rcfileTimeZone = TimeZone.getDefault().getID();
private boolean rcfileWriterValidate;
private HiveMetastoreAuthenticationType hiveMetastoreAuthenticationType = HiveMetastoreAuthenticationType.NONE;
@ -334,24 +337,6 @@ public class HiveConfig
return recursiveDirWalkerEnabled;
}
public DateTimeZone getDateTimeZone()
{
return DateTimeZone.forTimeZone(TimeZone.getTimeZone(timeZone));
}
@NotNull
public String getTimeZone()
{
return timeZone;
}
@Config("hive.time-zone")
public HiveConfig setTimeZone(String id)
{
this.timeZone = (id != null) ? id : TimeZone.getDefault().getID();
return this;
}
@NotNull
public DataSize getMaxSplitSize()
{
@ -947,6 +932,25 @@ public class HiveConfig
return this;
}
public DateTimeZone getRcfileDateTimeZone()
{
return DateTimeZone.forTimeZone(TimeZone.getTimeZone(rcfileTimeZone));
}
@NotNull
public String getRcfileTimeZone()
{
return rcfileTimeZone;
}
@Config("hive.rcfile.time-zone")
@ConfigDescription("Time zone for RCFile binary read and write")
public HiveConfig setRcfileTimeZone(String rcfileTimeZone)
{
this.rcfileTimeZone = rcfileTimeZone;
return this;
}
public boolean isRcfileWriterValidate()
{
return rcfileWriterValidate;
@ -988,6 +992,44 @@ public class HiveConfig
return this;
}
public DateTimeZone getOrcLegacyDateTimeZone()
{
return DateTimeZone.forTimeZone(TimeZone.getTimeZone(orcLegacyTimeZone));
}
@NotNull
public String getOrcLegacyTimeZone()
{
return orcLegacyTimeZone;
}
@Config("hive.orc.time-zone")
@ConfigDescription("Time zone for legacy ORC files that do not contain a time zone")
public HiveConfig setOrcLegacyTimeZone(String orcLegacyTimeZone)
{
this.orcLegacyTimeZone = orcLegacyTimeZone;
return this;
}
public DateTimeZone getParquetDateTimeZone()
{
return DateTimeZone.forTimeZone(TimeZone.getTimeZone(parquetTimeZone));
}
@NotNull
public String getParquetTimeZone()
{
return parquetTimeZone;
}
@Config("hive.parquet.time-zone")
@ConfigDescription("Time zone for Parquet read and write")
public HiveConfig setParquetTimeZone(String parquetTimeZone)
{
this.parquetTimeZone = parquetTimeZone;
return this;
}
public boolean isUseParquetColumnNames()
{
return useParquetColumnNames;

29
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java
View File

@ -104,7 +104,6 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.OpenCSVSerde;
import org.apache.hadoop.mapred.JobConf;
import org.joda.time.DateTimeZone;
import java.io.File;
import java.io.IOException;
@ -141,7 +140,7 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Streams.stream;
import static io.prestosql.plugin.hive.HiveBucketing.containsTimestampBucketedV2;
import static io.prestosql.plugin.hive.HiveBucketing.bucketedOnTimestamp;
import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR;
import static io.prestosql.plugin.hive.HiveStorageFormat.ORC;
import static io.prestosql.plugin.hive.HiveTableProperties.IS_EXTERNAL_TABLE;
@ -218,11 +217,9 @@ public class HiveMetadata
private static final String CSV_QUOTE_KEY = OpenCSVSerde.QUOTECHAR;
private static final String CSV_ESCAPE_KEY = OpenCSVSerde.ESCAPECHAR;
private final boolean allowCorruptWritesForTesting;
protected final SemiTransactionalHiveMetastore metastore;
protected final HdfsEnvironment hdfsEnvironment;
private final HivePartitionManager partitionManager;
private final DateTimeZone timeZone;
protected final TypeManager typeManager;
protected final LocationService locationService;
private final JsonCodec<PartitionUpdate> partitionUpdateCodec;
@ -247,8 +244,6 @@ public class HiveMetadata
SemiTransactionalHiveMetastore metastore,
HdfsEnvironment hdfsEnvironment,
HivePartitionManager partitionManager,
DateTimeZone timeZone,
boolean allowCorruptWritesForTesting,
boolean writesToNonManagedTablesEnabled,
boolean createsOfNonManagedTablesEnabled,
boolean tableCreatesWithLocationAllowed,
@ -266,12 +261,9 @@ public class HiveMetadata
Optional<Duration> vacuumCollectorInterval,
ScheduledExecutorService hiveMetastoreClientService)
{
this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
this.metastore = requireNonNull(metastore, "metastore is null");
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
this.timeZone = requireNonNull(timeZone, "timeZone is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.locationService = requireNonNull(locationService, "locationService is null");
this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null");
@ -1162,7 +1154,6 @@ public class HiveMetadata
@Override
public ConnectorTableHandle beginStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle)
{
verifyJvmTimeZone();
SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName())
.orElseThrow(() -> new TableNotFoundException(tableName));
@ -1232,8 +1223,6 @@ public class HiveMetadata
@Override
public HiveOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
{
verifyJvmTimeZone();
if (getExternalLocation(tableMetadata.getProperties()) != null || isExternalTable(tableMetadata.getProperties())) {
throw new PrestoException(NOT_SUPPORTED, "External tables cannot be created using CREATE TABLE AS");
}
@ -1534,8 +1523,6 @@ public class HiveMetadata
private HiveInsertTableHandle beginInsertUpdateInternal(ConnectorSession session, ConnectorTableHandle tableHandle,
Optional<String> partition, HiveACIDWriteType writeType)
{
verifyJvmTimeZone();
HiveIdentity identity = new HiveIdentity(session);
SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
Table table = metastore.getTable(identity, tableName.getSchemaName(), tableName.getTableName())
@ -1855,7 +1842,6 @@ public class HiveMetadata
long rowCount = basicStatistics.getRowCount().orElseThrow(() -> new IllegalArgumentException("rowCount not present"));
Map<String, HiveColumnStatistics> columnStatistics = Statistics.fromComputedStatistics(
session,
timeZone,
computedColumnStatistics,
columnTypes,
rowCount);
@ -2458,8 +2444,8 @@ public class HiveMetadata
private Optional<ConnectorNewTableLayout> getInsertTableLayoutInternal(ConnectorSession session, Table table)
{
if (table.getStorage().getBucketProperty().isPresent()) {
if (containsTimestampBucketedV2(table.getStorage().getBucketProperty().get(), table)) {
throw new PrestoException(NOT_SUPPORTED, "Table bucketing version not supported for writing when bucketing on timestamp type");
if (bucketedOnTimestamp(table.getStorage().getBucketProperty().get(), table)) {
throw new PrestoException(NOT_SUPPORTED, "Writing to tables bucketed on timestamp not supported");
}
}
@ -2665,15 +2651,6 @@ public class HiveMetadata
return accessControlMetadata.listTablePrivileges(session, listTables(session, schemaTablePrefix));
}
protected void verifyJvmTimeZone()
{
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
throw new PrestoException(HiveErrorCode.HIVE_TIMEZONE_MISMATCH, format(
"To write Hive data, your JVM timezone must match the Hive storage timezone. Add -Duser.timezone=%s to your JVM arguments.",
timeZone.getID()));
}
}
public static HiveStorageFormat extractHiveStorageFormat(Table table)
{
StorageFormat storageFormat = table.getStorage().getStorageFormat();

21
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadataFactory.java
View File

@ -15,7 +15,6 @@ package io.prestosql.plugin.hive;
import io.airlift.concurrent.BoundedExecutor;
import io.airlift.json.JsonCodec;
import io.airlift.log.Logger;
import io.airlift.units.Duration;
import io.prestosql.plugin.hive.metastore.CachingHiveMetastore;
import io.prestosql.plugin.hive.metastore.HiveMetastore;
@ -24,7 +23,6 @@ import io.prestosql.plugin.hive.security.AccessControlMetadataFactory;
import io.prestosql.plugin.hive.statistics.MetastoreHiveStatisticsProvider;
import io.prestosql.plugin.hive.statistics.TableColumnStatistics;
import io.prestosql.spi.type.TypeManager;
import org.joda.time.DateTimeZone;
import javax.inject.Inject;
@ -41,12 +39,9 @@ import static java.util.Objects.requireNonNull;
public class HiveMetadataFactory
implements Supplier<TransactionalMetadata>
{
private static final Logger log = Logger.get(HiveMetadataFactory.class);
protected final Map<String, TableColumnStatistics> statsCache = new ConcurrentHashMap();
protected final Map<String, List<HivePartition>> samplePartitionCache = new ConcurrentHashMap();
private final boolean allowCorruptWritesForTesting;
private final boolean skipDeletionForAlter;
private final boolean skipTargetCleanupOnRollback;
private final boolean writesToNonManagedTablesEnabled;
@ -56,7 +51,6 @@ public class HiveMetadataFactory
private final HiveMetastore metastore;
private final HdfsEnvironment hdfsEnvironment;
private final HivePartitionManager partitionManager;
private final DateTimeZone timeZone;
private final TypeManager typeManager;
private final LocationService locationService;
private final JsonCodec<PartitionUpdate> partitionUpdateCodec;
@ -97,9 +91,7 @@ public class HiveMetadataFactory
metastore,
hdfsEnvironment,
partitionManager,
hiveConfig.getDateTimeZone(),
hiveConfig.getMaxConcurrentFileRenames(),
hiveConfig.getAllowCorruptWritesForTesting(),
hiveConfig.isSkipDeletionForAlter(),
hiveConfig.isSkipTargetCleanupOnRollback(),
hiveConfig.getWritesToNonManagedTablesEnabled(),
@ -129,9 +121,7 @@ public class HiveMetadataFactory
HiveMetastore metastore,
HdfsEnvironment hdfsEnvironment,
HivePartitionManager partitionManager,
DateTimeZone timeZone,
int maxConcurrentFileRenames,
boolean allowCorruptWritesForTesting,
boolean skipDeletionForAlter,
boolean skipTargetCleanupOnRollback,
boolean writesToNonManagedTablesEnabled,
@ -156,7 +146,6 @@ public class HiveMetadataFactory
Optional<Duration> vacuumCollectorInterval,
int hmsWriteBatchSize)
{
this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
this.skipDeletionForAlter = skipDeletionForAlter;
this.skipTargetCleanupOnRollback = skipTargetCleanupOnRollback;
this.writesToNonManagedTablesEnabled = writesToNonManagedTablesEnabled;
@ -167,7 +156,6 @@ public class HiveMetadataFactory
this.metastore = requireNonNull(metastore, "metastore is null");
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
this.timeZone = requireNonNull(timeZone, "timeZone is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.locationService = requireNonNull(locationService, "locationService is null");
this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null");
@ -177,13 +165,6 @@ public class HiveMetadataFactory
this.hiveTransactionHeartbeatInterval = requireNonNull(hiveTransactionHeartbeatInterval, "hiveTransactionHeartbeatInterval is null");
this.vacuumCleanupRecheckInterval = requireNonNull(vacuumCleanupRecheckInterval, "vacuumCleanupInterval is null");
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
log.warn("Hive writes are disabled. " +
"To write data to Hive, your JVM timezone must match the Hive storage timezone. " +
"Add -Duser.timezone=%s to your JVM arguments",
timeZone.getID());
}
renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
this.hiveVacuumService = requireNonNull(hiveVacuumService, "hiveVacuumService is null");
this.heartbeatService = requireNonNull(heartbeatService, "heartbeatService is null");
@ -215,8 +196,6 @@ public class HiveMetadataFactory
metastore,
hdfsEnvironment,
partitionManager,
timeZone,
allowCorruptWritesForTesting,
writesToNonManagedTablesEnabled,
createsOfNonManagedTablesEnabled,
tableCreatesWithLocationAllowed,

2
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSink.java
View File

@ -524,7 +524,7 @@ public class HivePageSink
for (int i = 0; i < partitionKeys.size(); i++) {
HivePartitionKey partitionKey = partitionKeys.get(i);
Type type = partitionTypes.get(i);
Object partitionColumnValue = HiveUtil.typedPartitionKey(partitionKey.getValue(), type, partitionKey.getName(), null);
Object partitionColumnValue = HiveUtil.typedPartitionKey(partitionKey.getValue(), type, partitionKey.getName());
RunLengthEncodedBlock block = RunLengthEncodedBlock.create(type, partitionColumnValue, 1);
type.appendTo(block, 0, builder.getBlockBuilder(i));
}

4
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSinkProvider.java
View File

@ -39,6 +39,7 @@ import io.prestosql.spi.connector.ConnectorUpdateTableHandle;
import io.prestosql.spi.connector.ConnectorVacuumTableHandle;
import io.prestosql.spi.type.TypeManager;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.joda.time.DateTimeZone;
import javax.inject.Inject;
@ -74,6 +75,7 @@ public class HivePageSinkProvider
private final HiveWriterStats hiveWriterStats;
private final OrcFileWriterFactory orcFileWriterFactory;
private final long perTransactionMetastoreCacheMaximumSize;
private final DateTimeZone parquetTimeZone;
@Inject
public HivePageSinkProvider(
@ -111,6 +113,7 @@ public class HivePageSinkProvider
this.hiveWriterStats = requireNonNull(hiveWriterStats, "stats is null");
this.orcFileWriterFactory = requireNonNull(orcFileWriterFactory, "orcFileWriterFactory is null");
this.perTransactionMetastoreCacheMaximumSize = config.getPerTransactionMetastoreCacheMaximumSize();
this.parquetTimeZone = config.getParquetDateTimeZone();
}
@Override
@ -186,6 +189,7 @@ public class HivePageSinkProvider
writerSortBufferSize,
maxOpenSortFiles,
immutablePartitions,
parquetTimeZone,
session,
nodeManager,
eventClient,

5
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSource.java
View File

@ -39,7 +39,6 @@ import io.prestosql.spi.type.TypeManager;
import io.prestosql.spi.type.TypeUtils;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.UncheckedIOException;
@ -88,14 +87,12 @@ public class HivePageSource
public HivePageSource(
List<ColumnMapping> columnMappings,
Optional<BucketAdaptation> bucketAdaptation,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
ConnectorPageSource delegate,
Optional<DynamicFilterSupplier> dynamicFilterSupplier,
ConnectorSession session,
List<HivePartitionKey> partitionKeys)
{
requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.delegate = requireNonNull(delegate, "delegate is null");
@ -129,7 +126,7 @@ public class HivePageSource
}
if (columnMapping.getKind() == PREFILLED) {
prefilledValues[columnIndex] = typedPartitionKey(columnMapping.getPrefilledValue(), type, name, hiveStorageTimeZone);
prefilledValues[columnIndex] = typedPartitionKey(columnMapping.getPrefilledValue(), type, name);
}
}
this.coercers = coercers.build();

2
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceFactory.java
View File

@ -21,7 +21,6 @@ import io.prestosql.spi.heuristicindex.SplitMetadata;
import io.prestosql.spi.predicate.TupleDomain;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.joda.time.DateTimeZone;
import java.util.List;
import java.util.Optional;
@ -39,7 +38,6 @@ public interface HivePageSourceFactory
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
Optional<DynamicFilterSupplier> dynamicFilterSupplier,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,

14
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java
View File

@ -51,7 +51,6 @@ import io.prestosql.spi.type.TypeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.eclipse.jetty.util.URIUtil;
import org.joda.time.DateTimeZone;
import javax.inject.Inject;
@ -82,7 +81,6 @@ import static java.util.stream.Collectors.toList;
public class HivePageSourceProvider
implements ConnectorPageSourceProvider
{
private final DateTimeZone hiveStorageTimeZone;
private final HdfsEnvironment hdfsEnvironment;
private final Set<HiveRecordCursorProvider> cursorProviders;
private final TypeManager typeManager;
@ -104,7 +102,6 @@ public class HivePageSourceProvider
Set<HiveSelectivePageSourceFactory> selectivePageSourceFactories)
{
requireNonNull(hiveConfig, "hiveConfig is null");
this.hiveStorageTimeZone = hiveConfig.getDateTimeZone();
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.cursorProviders = ImmutableSet.copyOf(requireNonNull(cursorProviders, "cursorProviders is null"));
this.pageSourceFactories = ImmutableSet.copyOf(
@ -215,7 +212,7 @@ public class HivePageSourceProvider
*/
if (hiveTable.isSuitableToPush()) {
return createSelectivePageSource(selectivePageSourceFactories, configuration,
session, hiveSplit, assignUniqueIndicesToPartitionColumns(hiveColumns), hiveStorageTimeZone, typeManager,
session, hiveSplit, assignUniqueIndicesToPartitionColumns(hiveColumns), typeManager,
dynamicFilterSupplier, hiveSplit.getDeleteDeltaLocations(),
hiveSplit.getStartRowOffsetOfFile(),
indexOptional, hiveSplit.isCacheable(),
@ -241,7 +238,6 @@ public class HivePageSourceProvider
hiveTable.getCompactEffectivePredicate().intersect(predicate),
hiveColumns,
hiveSplit.getPartitionKeys(),
hiveStorageTimeZone,
typeManager,
hiveSplit.getColumnCoercions(),
hiveSplit.getBucketConversion(),
@ -303,7 +299,6 @@ public class HivePageSourceProvider
ConnectorSession session,
HiveSplit split,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
Optional<DynamicFilterSupplier> dynamicFilterSupplier,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
@ -369,7 +364,6 @@ public class HivePageSourceProvider
outputColumns,
effectivePredicate,
additionPredicates,
hiveStorageTimeZone,
deleteDeltaLocations,
startRowOffsetOfFile,
indexes,
@ -381,7 +375,6 @@ public class HivePageSourceProvider
return new HivePageSource(
columnMappings,
Optional.empty(),
hiveStorageTimeZone,
typeManager,
pageSource.get(),
dynamicFilterSupplier,
@ -407,7 +400,6 @@ public class HivePageSourceProvider
TupleDomain<HiveColumnHandle> effectivePredicate,
List<HiveColumnHandle> hiveColumns,
List<HivePartitionKey> partitionKeys,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
Map<Integer, HiveType> columnCoercions,
Optional<HiveSplit.BucketConversion> bucketConversion,
@ -445,7 +437,6 @@ public class HivePageSourceProvider
schema,
toColumnHandles(regularAndInterimColumnMappings, true),
effectivePredicate,
hiveStorageTimeZone,
dynamicFilterSupplier,
deleteDeltaLocations,
startRowOffsetOfFile,
@ -458,7 +449,6 @@ public class HivePageSourceProvider
new HivePageSource(
columnMappings,
bucketAdaptation,
hiveStorageTimeZone,
typeManager,
pageSource.get(),
dynamicFilterSupplier,
@ -481,7 +471,6 @@ public class HivePageSourceProvider
schema,
toColumnHandles(regularAndInterimColumnMappings, doCoercion),
effectivePredicate,
hiveStorageTimeZone,
typeManager,
s3SelectPushdownEnabled,
customSplitInfo);
@ -510,7 +499,6 @@ public class HivePageSourceProvider
HiveRecordCursor hiveRecordCursor = new HiveRecordCursor(
columnMappings,
hiveStorageTimeZone,
typeManager,
delegate);
List<Type> columnTypes = hiveColumns.stream()

12
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePartitionManager.java
View File

@ -50,7 +50,6 @@ import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeManager;
import io.prestosql.spi.type.VarcharType;
import org.apache.hadoop.hive.common.FileUtils;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
@ -80,7 +79,6 @@ public class HivePartitionManager
{
private static final String PARTITION_VALUE_WILDCARD = "";
private final DateTimeZone timeZone;
private final int maxPartitions;
private final boolean assumeCanonicalPartitionKeys;
private final int domainCompactionThreshold;
@ -93,7 +91,6 @@ public class HivePartitionManager
{
this(
typeManager,
hiveConfig.getDateTimeZone(),
hiveConfig.getMaxPartitionsPerScan(),
hiveConfig.isAssumeCanonicalPartitionKeys(),
hiveConfig.getDomainCompactionThreshold());
@ -101,12 +98,10 @@ public class HivePartitionManager
public HivePartitionManager(
TypeManager typeManager,
DateTimeZone timeZone,
int maxPartitions,
boolean assumeCanonicalPartitionKeys,
int domainCompactionThreshold)
{
this.timeZone = requireNonNull(timeZone, "timeZone is null");
checkArgument(maxPartitions >= 1, "maxPartitions must be at least 1");
this.maxPartitions = maxPartitions;
this.assumeCanonicalPartitionKeys = assumeCanonicalPartitionKeys;
@ -267,7 +262,7 @@ public class HivePartitionManager
TupleDomain<ColumnHandle> constraintSummary,
Predicate<Map<ColumnHandle, NullableValue>> constraint)
{
HivePartition partition = parsePartition(tableName, partitionId, partitionColumns, partitionColumnTypes, timeZone);
HivePartition partition = parsePartition(tableName, partitionId, partitionColumns, partitionColumnTypes);
if (partitionMatches(partitionColumns, constraintSummary, constraint, partition)) {
return Optional.of(partition);
@ -360,14 +355,13 @@ public class HivePartitionManager
SchemaTableName tableName,
String partitionName,
List<HiveColumnHandle> partitionColumns,
List<Type> partitionColumnTypes,
DateTimeZone timeZone)
List<Type> partitionColumnTypes)
{
List<String> partitionValues = extractPartitionValues(partitionName);
ImmutableMap.Builder<ColumnHandle, NullableValue> builder = ImmutableMap.builder();
for (int i = 0; i < partitionColumns.size(); i++) {
HiveColumnHandle column = partitionColumns.get(i);
NullableValue parsedValue = parsePartitionValue(partitionName, partitionValues.get(i), partitionColumnTypes.get(i), timeZone);
NullableValue parsedValue = parsePartitionValue(partitionName, partitionValues.get(i), partitionColumnTypes.get(i));
builder.put(column, parsedValue);
}
Map<ColumnHandle, NullableValue> values = builder.build();

5
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveRecordCursor.java
View File

@ -20,7 +20,6 @@ import io.prestosql.spi.connector.RecordCursor;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeManager;
import org.joda.time.DateTimeZone;
import java.util.List;
@ -72,13 +71,11 @@ public class HiveRecordCursor
public HiveRecordCursor(
List<HivePageSourceProvider.ColumnMapping> columnMappings,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
RecordCursor delegate)
{
requireNonNull(columnMappings, "columns is null");
requireNonNull(typeManager, "typeManager is null");
requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.delegate = requireNonNull(delegate, "delegate is null");
this.columnMappings = columnMappings;
@ -139,7 +136,7 @@ public class HiveRecordCursor
longs[columnIndex] = datePartitionKey(columnValue, name);
}
else if (TIMESTAMP.equals(type)) {
longs[columnIndex] = timestampPartitionKey(columnValue, hiveStorageTimeZone, name);
longs[columnIndex] = timestampPartitionKey(columnValue, name);
}
else if (isShortDecimal(type)) {
longs[columnIndex] = shortDecimalPartitionKey(columnValue, (DecimalType) type, name);

2
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveRecordCursorProvider.java
View File

@ -19,7 +19,6 @@ import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.spi.type.TypeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.joda.time.DateTimeZone;
import java.util.List;
import java.util.Map;
@ -38,7 +37,6 @@ public interface HiveRecordCursorProvider
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
boolean s3SelectPushdownEnabled,
Map<String, String> customSplitInfo);

2
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSelectivePageSourceFactory.java
View File

@ -20,7 +20,6 @@ import io.prestosql.spi.heuristicindex.IndexMetadata;
import io.prestosql.spi.predicate.TupleDomain;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.joda.time.DateTimeZone;
import java.util.List;
import java.util.Map;
@ -42,7 +41,6 @@ public interface HiveSelectivePageSourceFactory
List<Integer> outputColumns,
TupleDomain<HiveColumnHandle> domainPredicate,
Optional<List<TupleDomain<HiveColumnHandle>>> additionPredicates,
DateTimeZone hiveStorageTimeZone,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
Optional<List<IndexMetadata>> indexes,

2
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java
View File

@ -510,7 +510,7 @@ class HiveSplitSource
nullableValue = NullableValue.asNull(columnMetadata.getType());
}
else {
nullableValue = HiveUtil.parsePartitionValue(columnMetadata.getName(), partitionStringValue, columnMetadata.getType(), hiveConfig.getDateTimeZone());
nullableValue = HiveUtil.parsePartitionValue(columnMetadata.getName(), partitionStringValue, columnMetadata.getType());
}
return domain.includesNullableValue(nullableValue.getValue());
});

21
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveUtil.java
View File

@ -85,7 +85,6 @@ import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hudi.hadoop.realtime.HoodieRealtimeFileSplit;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.DateTimeFormatterBuilder;
@ -123,7 +122,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.transform;
import static io.prestosql.plugin.hive.HiveBucketing.containsTimestampBucketedV2;
import static io.prestosql.plugin.hive.HiveBucketing.bucketedOnTimestamp;
import static io.prestosql.plugin.hive.HiveColumnHandle.bucketColumnHandle;
import static io.prestosql.plugin.hive.util.CustomSplitConversionUtils.recreateSplitWithCustomInfo;
import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
@ -378,9 +377,9 @@ public final class HiveUtil
return TimeUnit.MILLISECONDS.toDays(millis);
}
public static long parseHiveTimestamp(String value, DateTimeZone timeZone)
public static long parseHiveTimestamp(String value)
{
return HIVE_TIMESTAMP_PARSER.withZone(timeZone).parseMillis(value);
return HIVE_TIMESTAMP_PARSER.parseMillis(value);
}
public static boolean isSplittable(InputFormat<?, ?> inputFormat, FileSystem fileSystem, Path path)
@ -525,7 +524,7 @@ public final class HiveUtil
isCharType(type);
}
public static NullableValue parsePartitionValue(String partitionName, String value, Type type, DateTimeZone timeZone)
public static NullableValue parsePartitionValue(String partitionName, String value, Type type)
{
verifyPartitionTypeSupported(partitionName, type);
@ -611,7 +610,7 @@ public final class HiveUtil
if (isNull) {
return NullableValue.asNull(TIMESTAMP);
}
return NullableValue.of(TIMESTAMP, timestampPartitionKey(value, timeZone, partitionName));
return NullableValue.of(TIMESTAMP, timestampPartitionKey(value, partitionName));
}
if (REAL.equals(type)) {
@ -808,10 +807,10 @@ public final class HiveUtil
}
}
public static long timestampPartitionKey(String value, DateTimeZone zone, String name)
public static long timestampPartitionKey(String value, String name)
{
try {
return parseHiveTimestamp(value, zone);
return parseHiveTimestamp(value);
}
catch (IllegalArgumentException e) {
throw new PrestoException(HiveErrorCode.HIVE_INVALID_PARTITION_VALUE, format("Invalid partition value '%s' for TIMESTAMP partition key: %s", value, name));
@ -880,7 +879,7 @@ public final class HiveUtil
// add hidden columns
columns.add(HiveColumnHandle.pathColumnHandle());
if (table.getStorage().getBucketProperty().isPresent()) {
if (!containsTimestampBucketedV2(table.getStorage().getBucketProperty().get(), table)) {
if (!bucketedOnTimestamp(table.getStorage().getBucketProperty().get(), table)) {
columns.add(bucketColumnHandle());
}
}
@ -1025,7 +1024,7 @@ public final class HiveUtil
}
}
public static Object typedPartitionKey(String value, Type type, String name, DateTimeZone hiveStorageTimeZone)
public static Object typedPartitionKey(String value, Type type, String name)
{
byte[] bytes = value.getBytes(UTF_8);
@ -1063,7 +1062,7 @@ public final class HiveUtil
return datePartitionKey(value, name);
}
else if (type.equals(TIMESTAMP)) {
return timestampPartitionKey(value, hiveStorageTimeZone, name);
return timestampPartitionKey(value, name);
}
else if (isShortDecimal(type)) {
return shortDecimalPartitionKey(value, (DecimalType) type, name);

444
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriteUtils.java
View File

@ -24,6 +24,7 @@ import io.prestosql.plugin.hive.metastore.Partition;
import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
import io.prestosql.plugin.hive.metastore.Storage;
import io.prestosql.plugin.hive.metastore.Table;
import io.prestosql.plugin.hive.parquet.ParquetRecordWriter;
import io.prestosql.plugin.hive.s3.HiveS3Module;
import io.prestosql.plugin.hive.s3.PrestoS3FileSystem;
import io.prestosql.spi.Page;
@ -56,8 +57,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.ProtectMode;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
@ -66,37 +69,21 @@ import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.Serializer;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -104,7 +91,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import static com.google.common.base.Strings.padEnd;
import static com.google.common.io.BaseEncoding.base16;
@ -118,7 +104,6 @@ import static java.lang.Float.intBitsToFloat;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.UUID.randomUUID;
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.COMPRESSRESULT;
@ -150,7 +135,6 @@ import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveO
import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo;
import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo;
import static org.joda.time.DateTimeZone.UTC;
public final class HiveWriteUtils
{
@ -173,7 +157,7 @@ public final class HiveWriteUtils
try {
boolean compress = HiveConf.getBoolVar(conf, COMPRESSRESULT);
if (outputFormatName.equals(MapredParquetOutputFormat.class.getName())) {
return ParquetRecordWriterUtil.createParquetWriter(target, conf, properties, session);
return ParquetRecordWriter.create(target, conf, properties, session);
}
Object writer = Class.forName(outputFormatName).getConstructor().newInstance();
return ((HiveOutputFormat<?, ?>) writer).getHiveRecordWriter(conf, target, Text.class, compress, properties, Reporter.NULL);
@ -318,12 +302,10 @@ public final class HiveWriteUtils
return type.getSlice(block, position).getBytes();
}
if (DateType.DATE.equals(type)) {
long days = type.getLong(block, position);
return new Date(UTC.getMillisKeepLocal(DateTimeZone.getDefault(), TimeUnit.DAYS.toMillis(days)));
return Date.ofEpochDay(toIntExact(type.getLong(block, position)));
}
if (TimestampType.TIMESTAMP.equals(type)) {
long millisUtc = type.getLong(block, position);
return new Timestamp(millisUtc);
return Timestamp.ofEpochMilli(type.getLong(block, position));
}
if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
@ -742,335 +724,7 @@ public final class HiveWriteUtils
throw new IllegalArgumentException("unsupported type: " + type);
}
public static FieldSetter createFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
if (type.equals(BooleanType.BOOLEAN)) {
return new BooleanFieldSetter(rowInspector, row, field);
}
if (type.equals(BigintType.BIGINT)) {
return new BigintFieldBuilder(rowInspector, row, field);
}
if (type.equals(IntegerType.INTEGER)) {
return new IntFieldSetter(rowInspector, row, field);
}
if (type.equals(SmallintType.SMALLINT)) {
return new SmallintFieldSetter(rowInspector, row, field);
}
if (type.equals(TinyintType.TINYINT)) {
return new TinyintFieldSetter(rowInspector, row, field);
}
if (type.equals(RealType.REAL)) {
return new FloatFieldSetter(rowInspector, row, field);
}
if (type.equals(DoubleType.DOUBLE)) {
return new DoubleFieldSetter(rowInspector, row, field);
}
if (type instanceof VarcharType) {
return new VarcharFieldSetter(rowInspector, row, field, type);
}
if (type instanceof CharType) {
return new CharFieldSetter(rowInspector, row, field, type);
}
if (type.equals(VarbinaryType.VARBINARY)) {
return new BinaryFieldSetter(rowInspector, row, field);
}
if (type.equals(DateType.DATE)) {
return new DateFieldSetter(rowInspector, row, field);
}
if (type.equals(TimestampType.TIMESTAMP)) {
return new TimestampFieldSetter(rowInspector, row, field);
}
if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
return new DecimalFieldSetter(rowInspector, row, field, decimalType);
}
if (isArrayType(type)) {
return new ArrayFieldSetter(rowInspector, row, field, type.getTypeParameters().get(0));
}
if (isMapType(type)) {
return new MapFieldSetter(rowInspector, row, field, type.getTypeParameters().get(0), type.getTypeParameters().get(1));
}
if (isRowType(type)) {
return new RowFieldSetter(rowInspector, row, field, type.getTypeParameters());
}
throw new IllegalArgumentException("unsupported type: " + type);
}
public abstract static class FieldSetter
{
protected final SettableStructObjectInspector rowInspector;
protected final Object row;
protected final StructField field;
protected FieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
this.rowInspector = requireNonNull(rowInspector, "rowInspector is null");
this.row = requireNonNull(row, "row is null");
this.field = requireNonNull(field, "field is null");
}
public abstract <T> void setField(Block<T> block, int position);
}
private static class BooleanFieldSetter
extends FieldSetter
{
private final BooleanWritable value = new BooleanWritable();
public BooleanFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(BooleanType.BOOLEAN.getBoolean(block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class BigintFieldBuilder
extends FieldSetter
{
private final LongWritable value = new LongWritable();
public BigintFieldBuilder(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(BigintType.BIGINT.getLong(block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class IntFieldSetter
extends FieldSetter
{
private final IntWritable value = new IntWritable();
public IntFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(toIntExact(IntegerType.INTEGER.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class SmallintFieldSetter
extends FieldSetter
{
private final ShortWritable value = new ShortWritable();
public SmallintFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(Shorts.checkedCast(SmallintType.SMALLINT.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class TinyintFieldSetter
extends FieldSetter
{
private final ByteWritable value = new ByteWritable();
public TinyintFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(SignedBytes.checkedCast(TinyintType.TINYINT.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class DoubleFieldSetter
extends FieldSetter
{
private final DoubleWritable value = new DoubleWritable();
public DoubleFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(DoubleType.DOUBLE.getDouble(block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class FloatFieldSetter
extends FieldSetter
{
private final FloatWritable value = new FloatWritable();
public FloatFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(intBitsToFloat((int) RealType.REAL.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class VarcharFieldSetter
extends FieldSetter
{
private final Text value = new Text();
private final Type type;
public VarcharFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
super(rowInspector, row, field);
this.type = type;
}
@Override
public void setField(Block block, int position)
{
value.set(type.getSlice(block, position).getBytes());
rowInspector.setStructFieldData(row, field, value);
}
}
private static class CharFieldSetter
extends FieldSetter
{
private final Text value = new Text();
private final Type type;
public CharFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
super(rowInspector, row, field);
this.type = type;
}
@Override
public void setField(Block block, int position)
{
value.set(type.getSlice(block, position).getBytes());
rowInspector.setStructFieldData(row, field, value);
}
}
private static class BinaryFieldSetter
extends FieldSetter
{
private final BytesWritable value = new BytesWritable();
public BinaryFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
byte[] bytes = VarbinaryType.VARBINARY.getSlice(block, position).getBytes();
value.set(bytes, 0, bytes.length);
rowInspector.setStructFieldData(row, field, value);
}
}
private static class DateFieldSetter
extends FieldSetter
{
private final DateWritable value = new DateWritable();
public DateFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(toIntExact(DateType.DATE.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class TimestampFieldSetter
extends FieldSetter
{
private final TimestampWritable value = new TimestampWritable();
public TimestampFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
long millisUtc = TimestampType.TIMESTAMP.getLong(block, position);
value.setTime(millisUtc);
rowInspector.setStructFieldData(row, field, value);
}
}
private static class DecimalFieldSetter
extends FieldSetter
{
private final HiveDecimalWritable value = new HiveDecimalWritable();
private final DecimalType decimalType;
public DecimalFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, DecimalType decimalType)
{
super(rowInspector, row, field);
this.decimalType = decimalType;
}
@Override
public void setField(Block block, int position)
{
value.set(getHiveDecimal(decimalType, block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static HiveDecimal getHiveDecimal(DecimalType decimalType, Block block, int position)
public static HiveDecimal getHiveDecimal(DecimalType decimalType, Block block, int position)
{
BigInteger unscaledValue;
if (decimalType.isShort()) {
@ -1081,88 +735,4 @@ public final class HiveWriteUtils
}
return HiveDecimal.create(unscaledValue, decimalType.getScale());
}
private static class ArrayFieldSetter
extends FieldSetter
{
private final Type elementType;
public ArrayFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type elementType)
{
super(rowInspector, row, field);
this.elementType = requireNonNull(elementType, "elementType is null");
}
@Override
public <T> void setField(Block<T> block, int position)
{
Block<T> arrayBlock = block.getObject(position, Block.class);
List<Object> list = new ArrayList<>(arrayBlock.getPositionCount());
for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
Object element = getField(elementType, arrayBlock, i);
list.add(element);
}
rowInspector.setStructFieldData(row, field, list);
}
}
private static class MapFieldSetter
extends FieldSetter
{
private final Type keyType;
private final Type valueType;
public MapFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type keyType, Type valueType)
{
super(rowInspector, row, field);
this.keyType = requireNonNull(keyType, "keyType is null");
this.valueType = requireNonNull(valueType, "valueType is null");
}
@Override
public <T> void setField(Block<T> block, int position)
{
Block<T> mapBlock = block.getObject(position, Block.class);
Map<Object, Object> map = new HashMap<>(mapBlock.getPositionCount() * 2);
for (int i = 0; i < mapBlock.getPositionCount(); i += 2) {
Object key = getField(keyType, mapBlock, i);
Object value = getField(valueType, mapBlock, i + 1);
map.put(key, value);
}
rowInspector.setStructFieldData(row, field, map);
}
}
private static class RowFieldSetter
extends FieldSetter
{
private final List<Type> fieldTypes;
public RowFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, List<Type> fieldTypes)
{
super(rowInspector, row, field);
this.fieldTypes = ImmutableList.copyOf(fieldTypes);
}
@Override
public <T> void setField(Block<T> block, int position)
{
Block<T> rowBlock = block.getObject(position, Block.class);
// TODO reuse row object and use FieldSetters, like we do at the top level
// Ideally, we'd use the same recursive structure starting from the top, but
// this requires modeling row types in the same way we model table rows
// (multiple blocks vs all fields packed in a single block)
List<Object> value = new ArrayList<>(fieldTypes.size());
for (int i = 0; i < fieldTypes.size(); i++) {
Object element = getField(fieldTypes.get(i), rowBlock, i);
value.add(element);
}
rowInspector.setStructFieldData(row, field, value);
}
}
}

5
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriterFactory.java
View File

@ -60,6 +60,7 @@ import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hive.common.util.ReflectionUtil;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.security.Principal;
@ -141,6 +142,7 @@ public class HiveWriterFactory
private final int maxOpenSortFiles;
private final boolean immutablePartitions;
private final InsertExistingPartitionsBehavior insertExistingPartitionsBehavior;
private final DateTimeZone parquetTimeZone;
private final ConnectorSession session;
private final OptionalInt bucketCount;
@ -182,6 +184,7 @@ public class HiveWriterFactory
DataSize sortBufferSize,
int maxOpenSortFiles,
boolean immutablePartitions,
DateTimeZone parquetTimeZone,
ConnectorSession session,
NodeManager nodeManager,
EventClient eventClient,
@ -231,6 +234,7 @@ public class HiveWriterFactory
if (immutablePartitions) {
checkArgument(insertExistingPartitionsBehavior != InsertExistingPartitionsBehavior.APPEND, "insertExistingPartitionsBehavior cannot be APPEND");
}
this.parquetTimeZone = requireNonNull(parquetTimeZone, "parquetTimeZone is null");
this.acidWriteType = acidWriteType;
// divide input columns into partition and data columns
@ -654,6 +658,7 @@ public class HiveWriterFactory
partitionStorageFormat.getEstimatedWriterSystemMemoryUsage(),
conf,
typeManager,
parquetTimeZone,
session);
}
if (isTxnTable) {

3
presto-hive/src/main/java/io/prestosql/plugin/hive/OrcFileWriter.java
View File

@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.orc.impl.AcidStats;
import org.apache.orc.impl.OrcAcidUtils;
import org.joda.time.DateTimeZone;
import org.openjdk.jol.info.ClassLayout;
import java.io.IOException;
@ -102,7 +101,6 @@ public class OrcFileWriter
boolean writeLegacyVersion,
int[] fileInputColumnIndexes,
Map<String, String> metadata,
DateTimeZone hiveStorageTimeZone,
Optional<Supplier<OrcDataSource>> validationInputFactory,
OrcWriteValidationMode validationMode,
OrcWriterStats stats,
@ -122,7 +120,6 @@ public class OrcFileWriter
options,
writeLegacyVersion,
metadata,
hiveStorageTimeZone,
validationInputFactory.isPresent(),
validationMode,
stats,

6
presto-hive/src/main/java/io/prestosql/plugin/hive/OrcFileWriterFactory.java
View File

@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.orc.OrcConf;
import org.joda.time.DateTimeZone;
import org.weakref.jmx.Flatten;
import org.weakref.jmx.Managed;
@ -62,7 +61,6 @@ import static java.util.stream.Collectors.toList;
public class OrcFileWriterFactory
implements HiveFileWriterFactory
{
private final DateTimeZone hiveStorageTimeZone;
private final HdfsEnvironment hdfsEnvironment;
private final TypeManager typeManager;
private final NodeVersion nodeVersion;
@ -84,7 +82,6 @@ public class OrcFileWriterFactory
hdfsEnvironment,
typeManager,
nodeVersion,
requireNonNull(hiveConfig, "hiveConfig is null").getDateTimeZone(),
hiveConfig.isOrcWriteLegacyVersion(),
readStats,
requireNonNull(config, "config is null").toOrcWriterOptions());
@ -94,7 +91,6 @@ public class OrcFileWriterFactory
HdfsEnvironment hdfsEnvironment,
TypeManager typeManager,
NodeVersion nodeVersion,
DateTimeZone hiveStorageTimeZone,
boolean writeLegacyVersion,
FileFormatDataSourceStats readStats,
OrcWriterOptions orcWriterOptions)
@ -102,7 +98,6 @@ public class OrcFileWriterFactory
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null");
this.hiveStorageTimeZone = requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.writeLegacyVersion = writeLegacyVersion;
this.readStats = requireNonNull(readStats, "stats is null");
this.orcWriterOptions = requireNonNull(orcWriterOptions, "orcWriterOptions is null");
@ -224,7 +219,6 @@ public class OrcFileWriterFactory
.put(HiveMetadata.PRESTO_QUERY_ID_NAME, session.getQueryId())
.put("hive.acid.version", String.valueOf(AcidUtils.OrcAcidVersion.ORC_ACID_VERSION))
.build(),
hiveStorageTimeZone,
validationInputFactory,
HiveSessionProperties.getOrcOptimizedWriterValidateMode(session),
stats,

128
presto-hive/src/main/java/io/prestosql/plugin/hive/ParquetRecordWriterUtil.java
View File

@ -1,128 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.plugin.hive;
import com.google.common.base.Splitter;
import io.prestosql.spi.connector.ConnectorSession;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.parquet.convert.HiveSchemaConverter;
import org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport;
import org.apache.hadoop.hive.ql.io.parquet.write.ParquetRecordWriterWrapper;
import org.apache.hadoop.hive.serde2.io.ParquetHiveRecord;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetOutputFormat;
import org.apache.parquet.hadoop.ParquetRecordWriter;
import org.apache.parquet.schema.MessageType;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.List;
import java.util.Properties;
import static io.prestosql.plugin.hive.HiveSessionProperties.getParquetWriterBlockSize;
import static io.prestosql.plugin.hive.HiveSessionProperties.getParquetWriterPageSize;
import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfosFromTypeString;
public final class ParquetRecordWriterUtil
{
private static final Field REAL_WRITER_FIELD;
private static final Field INTERNAL_WRITER_FIELD;
private static final Field FILE_WRITER_FIELD;
static {
try {
REAL_WRITER_FIELD = ParquetRecordWriterWrapper.class.getDeclaredField("realWriter");
INTERNAL_WRITER_FIELD = ParquetRecordWriter.class.getDeclaredField("internalWriter");
FILE_WRITER_FIELD = INTERNAL_WRITER_FIELD.getType().getDeclaredField("parquetFileWriter");
REAL_WRITER_FIELD.setAccessible(true);
INTERNAL_WRITER_FIELD.setAccessible(true);
FILE_WRITER_FIELD.setAccessible(true);
}
catch (ReflectiveOperationException e) {
throw new AssertionError(e);
}
}
private ParquetRecordWriterUtil() {}
public static RecordWriter createParquetWriter(Path target, JobConf conf, Properties properties, ConnectorSession session)
throws IOException, ReflectiveOperationException
{
conf.setLong(ParquetOutputFormat.BLOCK_SIZE, getParquetWriterBlockSize(session).toBytes());
conf.setLong(ParquetOutputFormat.PAGE_SIZE, getParquetWriterPageSize(session).toBytes());
RecordWriter recordWriter = createParquetWriter(target, conf, properties);
Object realWriter = REAL_WRITER_FIELD.get(recordWriter);
Object internalWriter = INTERNAL_WRITER_FIELD.get(realWriter);
ParquetFileWriter fileWriter = (ParquetFileWriter) FILE_WRITER_FIELD.get(internalWriter);
return new RecordFileWriter.ExtendedRecordWriter()
{
private long length;
@Override
public long getWrittenBytes()
{
return length;
}
@Override
public void write(Writable value)
throws IOException
{
recordWriter.write(value);
length = fileWriter.getPos();
}
@Override
public void close(boolean abort)
throws IOException
{
recordWriter.close(abort);
if (!abort) {
length = target.getFileSystem(conf).getFileStatus(target).getLen();
}
}
};
}
private static RecordWriter createParquetWriter(Path target, JobConf conf, Properties properties)
throws IOException
{
if (conf.get(DataWritableWriteSupport.PARQUET_HIVE_SCHEMA) == null) {
List<String> columnNames = Splitter.on(',').splitToList(properties.getProperty(IOConstants.COLUMNS));
List<TypeInfo> columnTypes = getTypeInfosFromTypeString(properties.getProperty(IOConstants.COLUMNS_TYPES));
MessageType schema = HiveSchemaConverter.convert(columnNames, columnTypes);
setParquetSchema(conf, schema);
}
ParquetOutputFormat<ParquetHiveRecord> outputFormat = new ParquetOutputFormat<>(new DataWritableWriteSupport());
return new ParquetRecordWriterWrapper(outputFormat, conf, target.toString(), Reporter.NULL, properties);
}
public static void setParquetSchema(Configuration conf, MessageType schema)
{
DataWritableWriteSupport.setSchema(schema, conf);
}
}

12
presto-hive/src/main/java/io/prestosql/plugin/hive/RcFileFileWriterFactory.java
View File

@ -52,7 +52,7 @@ import static java.util.stream.Collectors.toList;
public class RcFileFileWriterFactory
implements HiveFileWriterFactory
{
private final DateTimeZone hiveStorageTimeZone;
private final DateTimeZone timeZone;
private final HdfsEnvironment hdfsEnvironment;
private final TypeManager typeManager;
private final NodeVersion nodeVersion;
@ -66,20 +66,20 @@ public class RcFileFileWriterFactory
HiveConfig hiveConfig,
FileFormatDataSourceStats stats)
{
this(hdfsEnvironment, typeManager, nodeVersion, requireNonNull(hiveConfig, "hiveConfig is null").getDateTimeZone(), stats);
this(hdfsEnvironment, typeManager, nodeVersion, requireNonNull(hiveConfig, "hiveConfig is null").getRcfileDateTimeZone(), stats);
}
public RcFileFileWriterFactory(
HdfsEnvironment hdfsEnvironment,
TypeManager typeManager,
NodeVersion nodeVersion,
DateTimeZone hiveStorageTimeZone,
DateTimeZone timeZone,
FileFormatDataSourceStats stats)
{
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null");
this.hiveStorageTimeZone = requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.timeZone = requireNonNull(timeZone, "timeZone is null");
this.stats = requireNonNull(stats, "stats is null");
}
@ -98,10 +98,10 @@ public class RcFileFileWriterFactory
RcFileEncoding rcFileEncoding;
if (LazyBinaryColumnarSerDe.class.getName().equals(storageFormat.getSerDe())) {
rcFileEncoding = new BinaryRcFileEncoding();
rcFileEncoding = new BinaryRcFileEncoding(timeZone);
}
else if (ColumnarSerDe.class.getName().equals(storageFormat.getSerDe())) {
rcFileEncoding = RcFilePageSourceFactory.createTextVectorEncoding(schema, hiveStorageTimeZone);
rcFileEncoding = RcFilePageSourceFactory.createTextVectorEncoding(schema);
}
else {
return Optional.empty();

15
presto-hive/src/main/java/io/prestosql/plugin/hive/RecordFileWriter.java
View File

@ -15,8 +15,9 @@ package io.prestosql.plugin.hive;
import com.google.common.collect.ImmutableList;
import io.airlift.units.DataSize;
import io.prestosql.plugin.hive.HiveWriteUtils.FieldSetter;
import io.prestosql.plugin.hive.metastore.StorageFormat;
import io.prestosql.plugin.hive.parquet.ParquetRecordWriter;
import io.prestosql.plugin.hive.util.FieldSetterFactory;
import io.prestosql.spi.Page;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.Block;
@ -31,6 +32,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.mapred.JobConf;
import org.joda.time.DateTimeZone;
import org.openjdk.jol.info.ClassLayout;
import java.io.IOException;
@ -43,7 +45,6 @@ import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR;
import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR;
import static io.prestosql.plugin.hive.HiveUtil.getColumnNames;
import static io.prestosql.plugin.hive.HiveUtil.getColumnTypes;
import static io.prestosql.plugin.hive.HiveWriteUtils.createFieldSetter;
import static io.prestosql.plugin.hive.HiveWriteUtils.createRecordWriter;
import static io.prestosql.plugin.hive.HiveWriteUtils.getRowColumnInspectors;
import static io.prestosql.plugin.hive.HiveWriteUtils.initializeSerializer;
@ -64,7 +65,7 @@ public class RecordFileWriter
private final SettableStructObjectInspector tableInspector;
private final List<StructField> structFields;
private final Object row;
private final FieldSetter[] setters;
private final FieldSetterFactory.FieldSetter[] setters;
private final long estimatedWriterSystemMemoryUsage;
private boolean committed;
@ -77,6 +78,7 @@ public class RecordFileWriter
DataSize estimatedWriterSystemMemoryUsage,
JobConf conf,
TypeManager typeManager,
DateTimeZone parquetTimeZone,
ConnectorSession session)
{
this.path = requireNonNull(path, "path is null");
@ -104,9 +106,12 @@ public class RecordFileWriter
row = tableInspector.create();
setters = new FieldSetter[structFields.size()];
DateTimeZone timeZone = (recordWriter instanceof ParquetRecordWriter) ? parquetTimeZone : DateTimeZone.UTC;
FieldSetterFactory fieldSetterFactory = new FieldSetterFactory(timeZone);
setters = new FieldSetterFactory.FieldSetter[structFields.size()];
for (int i = 0; i < setters.length; i++) {
setters[i] = createFieldSetter(tableInspector, row, structFields.get(i), fileColumnTypes.get(structFields.get(i).getFieldID()));
setters[i] = fieldSetterFactory.create(tableInspector, row, structFields.get(i), fileColumnTypes.get(structFields.get(i).getFieldID()));
}
this.estimatedWriterSystemMemoryUsage = estimatedWriterSystemMemoryUsage.toBytes();

4
presto-hive/src/main/java/io/prestosql/plugin/hive/S3SelectRecordCursor.java
View File

@ -18,7 +18,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.RecordReader;
import org.joda.time.DateTimeZone;
import java.util.ArrayList;
import java.util.List;
@ -50,10 +49,9 @@ class S3SelectRecordCursor<K, V extends Writable>
long totalBytes,
Properties splitSchema,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager)
{
super(configuration, path, recordReader, totalBytes, updateSplitSchema(splitSchema, columns), columns, hiveStorageTimeZone, typeManager);
super(configuration, path, recordReader, totalBytes, updateSplitSchema(splitSchema, columns), columns, typeManager);
}
// since s3select only returns the required column, not the whole columns

4
presto-hive/src/main/java/io/prestosql/plugin/hive/S3SelectRecordCursorProvider.java
View File

@ -23,7 +23,6 @@ import io.prestosql.spi.type.TypeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.joda.time.DateTimeZone;
import javax.inject.Inject;
@ -67,7 +66,6 @@ public class S3SelectRecordCursorProvider
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
boolean s3SelectPushdownEnabled,
Map<String, String> customSplitInfo)
@ -88,7 +86,7 @@ public class S3SelectRecordCursorProvider
IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(typeManager);
String ionSqlQuery = queryBuilder.buildSql(columns, effectivePredicate);
S3SelectLineRecordReader recordReader = new S3SelectCsvRecordReader(configuration, hiveConfig, path, start, length, schema, ionSqlQuery, s3ClientFactory);
return Optional.of(new S3SelectRecordCursor<>(configuration, path, recordReader, length, schema, columns, hiveStorageTimeZone, typeManager));
return Optional.of(new S3SelectRecordCursor<>(configuration, path, recordReader, length, schema, columns, typeManager));
}
// unsupported serdes

3
presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/thrift/ThriftHiveMetastore.java
View File

@ -1772,7 +1772,8 @@ public class ThriftHiveMetastore
new HiveObjectRef(TABLE, databaseName, tableName, null, null),
grantee.getName(),
ThriftMetastoreUtil.fromPrestoPrincipalType(grantee.getType()),
privilegeGrantInfo));
privilegeGrantInfo,
"SQL"));
}
return new PrivilegeBag(privilegeBagBuilder.build());
}

9
presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java
View File

@ -135,6 +135,7 @@ public class OrcPageSourceFactory
private final FileFormatDataSourceStats stats;
private final OrcCacheStore orcCacheStore;
private final int domainCompactionThreshold;
private final DateTimeZone legacyTimeZone;
@Inject
public OrcPageSourceFactory(TypeManager typeManager, HiveConfig config, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, OrcCacheStore orcCacheStore)
@ -146,6 +147,7 @@ public class OrcPageSourceFactory
this.stats = requireNonNull(stats, "stats is null");
this.orcCacheStore = orcCacheStore;
this.domainCompactionThreshold = config.getDomainCompactionThreshold();
this.legacyTimeZone = requireNonNull(config, "hiveConfig is null").getOrcLegacyDateTimeZone();
}
@Override
@ -159,7 +161,6 @@ public class OrcPageSourceFactory
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
Optional<DynamicFilterSupplier> dynamicFilters,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
@ -194,7 +195,7 @@ public class OrcPageSourceFactory
useOrcColumnNames,
isFullAcidTable(Maps.fromProperties(schema)),
effectivePredicate,
hiveStorageTimeZone,
legacyTimeZone,
typeManager,
getOrcMaxMergeDistance(session),
getOrcMaxBufferSize(session),
@ -228,7 +229,7 @@ public class OrcPageSourceFactory
boolean useOrcColumnNames,
boolean isFullAcid,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
DateTimeZone legacyFileTimeZone,
TypeManager typeManager,
DataSize maxMergeDistance,
DataSize maxBufferSize,
@ -394,7 +395,7 @@ public class OrcPageSourceFactory
predicateBuilder.build(),
start,
length,
hiveStorageTimeZone,
legacyFileTimeZone,
systemMemoryUsage,
INITIAL_BATCH_SIZE,
exception -> handleException(orcDataSource.getId(), exception),

2
presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSource.java
View File

@ -23,7 +23,6 @@ import io.prestosql.plugin.hive.HivePageSourceProvider.ColumnMapping;
import io.prestosql.spi.Page;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ConnectorPageSource;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeManager;
@ -53,7 +52,6 @@ public class OrcSelectivePageSource
OrcDataSource orcDataSource,
AggregatedMemoryContext systemMemoryContext,
FileFormatDataSourceStats stats,
ConnectorSession session,
List<ColumnMapping> columnMappings,
TypeManager typeManager)
{

10
presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSourceFactory.java
View File

@ -137,6 +137,7 @@ public class OrcSelectivePageSourceFactory
private final HdfsEnvironment hdfsEnvironment;
private final FileFormatDataSourceStats stats;
private final OrcCacheStore orcCacheStore;
private final DateTimeZone legacyTimeZone;
@Inject
public OrcSelectivePageSourceFactory(TypeManager typeManager, HiveConfig config, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, OrcCacheStore orcCacheStore)
@ -147,6 +148,7 @@ public class OrcSelectivePageSourceFactory
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.stats = requireNonNull(stats, "stats is null");
this.orcCacheStore = orcCacheStore;
this.legacyTimeZone = requireNonNull(config, "hiveConfig is null").getOrcLegacyDateTimeZone();
}
@Override
@ -163,7 +165,6 @@ public class OrcSelectivePageSourceFactory
List<Integer> outputColumns,
TupleDomain<HiveColumnHandle> domainPredicate,
Optional<List<TupleDomain<HiveColumnHandle>>> additionPredicates,
DateTimeZone hiveStorageTimeZone,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
Optional<List<IndexMetadata>> indexes,
@ -207,7 +208,7 @@ public class OrcSelectivePageSourceFactory
prefilledValues,
outputColumns,
domainPredicate,
hiveStorageTimeZone,
legacyTimeZone,
typeManager,
getOrcMaxMergeDistance(session),
getOrcMaxBufferSize(session),
@ -281,7 +282,7 @@ public class OrcSelectivePageSourceFactory
prefilledValues,
outputColumns,
domainPredicate,
hiveStorageTimeZone,
legacyTimeZone,
typeManager,
getOrcMaxMergeDistance(session),
getOrcMaxBufferSize(session),
@ -485,7 +486,7 @@ public class OrcSelectivePageSourceFactory
Map<Integer, Object> typedPrefilledValues = new HashMap<>();
for (Map.Entry prefilledValue : prefilledValues.entrySet()) {
typedPrefilledValues.put(Integer.valueOf(prefilledValue.getKey().toString()),
typedPartitionKey(prefilledValue.getValue().toString(), columnTypes.get(prefilledValue.getKey()), columnNames.get(prefilledValue.getKey()), hiveStorageTimeZone));
typedPartitionKey(prefilledValue.getValue().toString(), columnTypes.get(prefilledValue.getKey()), columnNames.get(prefilledValue.getKey())));
}
// Convert the predicate to each column id wise. Will be used to associate as filter with each column reader
@ -546,7 +547,6 @@ public class OrcSelectivePageSourceFactory
// isFullAcid && indexes.isPresent(),
systemMemoryUsage,
stats,
session,
columnMappings,
typeManager);
}

12
presto-hive/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java
View File

@ -28,6 +28,7 @@ import io.prestosql.plugin.hive.DeleteDeltaLocations;
import io.prestosql.plugin.hive.FileFormatDataSourceStats;
import io.prestosql.plugin.hive.HdfsEnvironment;
import io.prestosql.plugin.hive.HiveColumnHandle;
import io.prestosql.plugin.hive.HiveConfig;
import io.prestosql.plugin.hive.HivePageSourceFactory;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ConnectorPageSource;
@ -97,13 +98,15 @@ public class ParquetPageSourceFactory
private final TypeManager typeManager;
private final HdfsEnvironment hdfsEnvironment;
private final FileFormatDataSourceStats stats;
private final DateTimeZone timeZone;
@Inject
public ParquetPageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats)
public ParquetPageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, HiveConfig hiveConfig)
{
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.stats = requireNonNull(stats, "stats is null");
timeZone = requireNonNull(hiveConfig, "hiveConfig is null").getParquetDateTimeZone();
}
@Override
@ -117,7 +120,6 @@ public class ParquetPageSourceFactory
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
Optional<DynamicFilterSupplier> dynamicFilter,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
@ -147,7 +149,8 @@ public class ParquetPageSourceFactory
getParquetMaxReadBlockSize(session),
typeManager,
effectivePredicate,
stats));
stats,
timeZone));
}
public static ParquetPageSource createParquetPageSource(
@ -165,7 +168,8 @@ public class ParquetPageSourceFactory
DataSize maxReadBlockSize,
TypeManager typeManager,
TupleDomain<HiveColumnHandle> effectivePredicate,
FileFormatDataSourceStats stats)
FileFormatDataSourceStats stats,
DateTimeZone timeZone)
{
AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext();

108
presto-hive/src/main/java/io/prestosql/plugin/hive/parquet/ParquetRecordWriter.java Normal file
View File

@ -0,0 +1,108 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.plugin.hive.parquet;
import io.prestosql.plugin.hive.RecordFileWriter.ExtendedRecordWriter;
import io.prestosql.spi.connector.ConnectorSession;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat;
import org.apache.hadoop.hive.ql.io.parquet.write.ParquetRecordWriterWrapper;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetOutputFormat;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Properties;
import static io.prestosql.plugin.hive.HiveSessionProperties.getParquetWriterBlockSize;
import static io.prestosql.plugin.hive.HiveSessionProperties.getParquetWriterPageSize;
import static java.util.Objects.requireNonNull;
public final class ParquetRecordWriter
implements ExtendedRecordWriter
{
private static final Field REAL_WRITER_FIELD;
private static final Field INTERNAL_WRITER_FIELD;
private static final Field FILE_WRITER_FIELD;
static {
try {
REAL_WRITER_FIELD = ParquetRecordWriterWrapper.class.getDeclaredField("realWriter");
INTERNAL_WRITER_FIELD = org.apache.parquet.hadoop.ParquetRecordWriter.class.getDeclaredField("internalWriter");
FILE_WRITER_FIELD = INTERNAL_WRITER_FIELD.getType().getDeclaredField("parquetFileWriter");
REAL_WRITER_FIELD.setAccessible(true);
INTERNAL_WRITER_FIELD.setAccessible(true);
FILE_WRITER_FIELD.setAccessible(true);
}
catch (ReflectiveOperationException e) {
throw new AssertionError(e);
}
}
public static RecordWriter create(Path target, JobConf conf, Properties properties, ConnectorSession session)
throws IOException, ReflectiveOperationException
{
conf.setLong(ParquetOutputFormat.BLOCK_SIZE, getParquetWriterBlockSize(session).toBytes());
conf.setLong(ParquetOutputFormat.PAGE_SIZE, getParquetWriterPageSize(session).toBytes());
RecordWriter recordWriter = new MapredParquetOutputFormat()
.getHiveRecordWriter(conf, target, Text.class, false, properties, Reporter.NULL);
Object realWriter = REAL_WRITER_FIELD.get(recordWriter);
Object internalWriter = INTERNAL_WRITER_FIELD.get(realWriter);
ParquetFileWriter fileWriter = (ParquetFileWriter) FILE_WRITER_FIELD.get(internalWriter);
return new ParquetRecordWriter(recordWriter, fileWriter);
}
private final RecordWriter recordWriter;
private final ParquetFileWriter fileWriter;
private long length;
private ParquetRecordWriter(RecordWriter recordWriter, ParquetFileWriter fileWriter)
{
this.recordWriter = requireNonNull(recordWriter, "recordWriter is null");
this.fileWriter = requireNonNull(fileWriter, "fileWriter is null");
}
@Override
public long getWrittenBytes()
{
return length;
}
@Override
public void write(Writable value)
throws IOException
{
recordWriter.write(value);
length = fileWriter.getPos();
}
@Override
public void close(boolean abort)
throws IOException
{
recordWriter.close(abort);
if (!abort) {
length = fileWriter.getPos();
}
}
}

13
presto-hive/src/main/java/io/prestosql/plugin/hive/rcfile/RcFilePageSourceFactory.java
View File

@ -22,6 +22,7 @@ import io.prestosql.plugin.hive.DeleteDeltaLocations;
import io.prestosql.plugin.hive.FileFormatDataSourceStats;
import io.prestosql.plugin.hive.HdfsEnvironment;
import io.prestosql.plugin.hive.HiveColumnHandle;
import io.prestosql.plugin.hive.HiveConfig;
import io.prestosql.plugin.hive.HivePageSourceFactory;
import io.prestosql.rcfile.AircompressorCodecFactory;
import io.prestosql.rcfile.HadoopCodecFactory;
@ -86,13 +87,15 @@ public class RcFilePageSourceFactory
private final TypeManager typeManager;
private final HdfsEnvironment hdfsEnvironment;
private final FileFormatDataSourceStats stats;
private final DateTimeZone timeZone;
@Inject
public RcFilePageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats)
public RcFilePageSourceFactory(TypeManager typeManager, HdfsEnvironment hdfsEnvironment, FileFormatDataSourceStats stats, HiveConfig hiveConfig)
{
this.typeManager = requireNonNull(typeManager, "typeManager is null");
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
this.stats = requireNonNull(stats, "stats is null");
this.timeZone = requireNonNull(hiveConfig, "hiveConfig is null").getRcfileDateTimeZone();
}
@Override
@ -106,7 +109,6 @@ public class RcFilePageSourceFactory
Properties schema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
Optional<DynamicFilterSupplier> dynamicFilters,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
@ -118,10 +120,10 @@ public class RcFilePageSourceFactory
RcFileEncoding rcFileEncoding;
String deserializerClassName = getDeserializerClassName(schema);
if (deserializerClassName.equals(LazyBinaryColumnarSerDe.class.getName())) {
rcFileEncoding = new BinaryRcFileEncoding();
rcFileEncoding = new BinaryRcFileEncoding(timeZone);
}
else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) {
rcFileEncoding = createTextVectorEncoding(schema, hiveStorageTimeZone);
rcFileEncoding = createTextVectorEncoding(schema);
}
else {
return Optional.empty();
@ -188,7 +190,7 @@ public class RcFilePageSourceFactory
return format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, t.getMessage());
}
public static TextRcFileEncoding createTextVectorEncoding(Properties schema, DateTimeZone hiveStorageTimeZone)
public static TextRcFileEncoding createTextVectorEncoding(Properties schema)
{
// separators
int nestingLevels;
@ -227,7 +229,6 @@ public class RcFilePageSourceFactory
}
return new TextRcFileEncoding(
hiveStorageTimeZone,
nullSequence,
separators,
escapeByte,

488
presto-hive/src/main/java/io/prestosql/plugin/hive/util/FieldSetterFactory.java Normal file
View File

@ -0,0 +1,488 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.plugin.hive.util;
import com.google.common.collect.ImmutableList;
import com.google.common.primitives.Shorts;
import com.google.common.primitives.SignedBytes;
import io.prestosql.plugin.hive.HiveWriteUtils;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.type.BigintType;
import io.prestosql.spi.type.BooleanType;
import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.DateType;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.DoubleType;
import io.prestosql.spi.type.IntegerType;
import io.prestosql.spi.type.RealType;
import io.prestosql.spi.type.SmallintType;
import io.prestosql.spi.type.TimestampType;
import io.prestosql.spi.type.TinyintType;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.VarbinaryType;
import io.prestosql.spi.type.VarcharType;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.joda.time.DateTimeZone;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static io.prestosql.plugin.hive.HiveUtil.isArrayType;
import static io.prestosql.plugin.hive.HiveUtil.isMapType;
import static io.prestosql.plugin.hive.HiveUtil.isRowType;
import static io.prestosql.plugin.hive.HiveWriteUtils.getHiveDecimal;
import static java.lang.Float.intBitsToFloat;
import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
public final class FieldSetterFactory
{
private final DateTimeZone timeZone;
public FieldSetterFactory(DateTimeZone timeZone)
{
this.timeZone = requireNonNull(timeZone, "timeZone is null");
}
public FieldSetter create(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
if (type.equals(BooleanType.BOOLEAN)) {
return new BooleanFieldSetter(rowInspector, row, field);
}
if (type.equals(BigintType.BIGINT)) {
return new BigintFieldBuilder(rowInspector, row, field);
}
if (type.equals(IntegerType.INTEGER)) {
return new IntFieldSetter(rowInspector, row, field);
}
if (type.equals(SmallintType.SMALLINT)) {
return new SmallintFieldSetter(rowInspector, row, field);
}
if (type.equals(TinyintType.TINYINT)) {
return new TinyintFieldSetter(rowInspector, row, field);
}
if (type.equals(RealType.REAL)) {
return new FloatFieldSetter(rowInspector, row, field);
}
if (type.equals(DoubleType.DOUBLE)) {
return new DoubleFieldSetter(rowInspector, row, field);
}
if (type instanceof VarcharType) {
return new VarcharFieldSetter(rowInspector, row, field, type);
}
if (type instanceof CharType) {
return new CharFieldSetter(rowInspector, row, field, type);
}
if (type.equals(VarbinaryType.VARBINARY)) {
return new BinaryFieldSetter(rowInspector, row, field);
}
if (type.equals(DateType.DATE)) {
return new DateFieldSetter(rowInspector, row, field);
}
if (type.equals(TimestampType.TIMESTAMP)) {
return new TimestampFieldSetter(rowInspector, row, field, timeZone);
}
if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
return new DecimalFieldSetter(rowInspector, row, field, decimalType);
}
if (isArrayType(type)) {
return new ArrayFieldSetter(rowInspector, row, field, type.getTypeParameters().get(0));
}
if (isMapType(type)) {
return new MapFieldSetter(rowInspector, row, field, type.getTypeParameters().get(0), type.getTypeParameters().get(1));
}
if (isRowType(type)) {
return new RowFieldSetter(rowInspector, row, field, type.getTypeParameters());
}
throw new IllegalArgumentException("unsupported type: " + type);
}
public abstract static class FieldSetter
{
protected final SettableStructObjectInspector rowInspector;
protected final Object row;
protected final StructField field;
private FieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
this.rowInspector = requireNonNull(rowInspector, "rowInspector is null");
this.row = requireNonNull(row, "row is null");
this.field = requireNonNull(field, "field is null");
}
public abstract void setField(Block block, int position);
}
private static class BooleanFieldSetter
extends FieldSetter
{
private final BooleanWritable value = new BooleanWritable();
public BooleanFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(BooleanType.BOOLEAN.getBoolean(block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class BigintFieldBuilder
extends FieldSetter
{
private final LongWritable value = new LongWritable();
public BigintFieldBuilder(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(BigintType.BIGINT.getLong(block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class IntFieldSetter
extends FieldSetter
{
private final IntWritable value = new IntWritable();
public IntFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(toIntExact(IntegerType.INTEGER.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class SmallintFieldSetter
extends FieldSetter
{
private final ShortWritable value = new ShortWritable();
public SmallintFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(Shorts.checkedCast(SmallintType.SMALLINT.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class TinyintFieldSetter
extends FieldSetter
{
private final ByteWritable value = new ByteWritable();
public TinyintFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(SignedBytes.checkedCast(TinyintType.TINYINT.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class DoubleFieldSetter
extends FieldSetter
{
private final DoubleWritable value = new DoubleWritable();
public DoubleFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(DoubleType.DOUBLE.getDouble(block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class FloatFieldSetter
extends FieldSetter
{
private final FloatWritable value = new FloatWritable();
public FloatFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(intBitsToFloat((int) RealType.REAL.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class VarcharFieldSetter
extends FieldSetter
{
private final Text value = new Text();
private final Type type;
public VarcharFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
super(rowInspector, row, field);
this.type = type;
}
@Override
public void setField(Block block, int position)
{
value.set(type.getSlice(block, position).getBytes());
rowInspector.setStructFieldData(row, field, value);
}
}
private static class CharFieldSetter
extends FieldSetter
{
private final Text value = new Text();
private final Type type;
public CharFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type type)
{
super(rowInspector, row, field);
this.type = type;
}
@Override
public void setField(Block block, int position)
{
value.set(type.getSlice(block, position).getBytes());
rowInspector.setStructFieldData(row, field, value);
}
}
private static class BinaryFieldSetter
extends FieldSetter
{
private final BytesWritable value = new BytesWritable();
public BinaryFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
byte[] bytes = VarbinaryType.VARBINARY.getSlice(block, position).getBytes();
value.set(bytes, 0, bytes.length);
rowInspector.setStructFieldData(row, field, value);
}
}
private static class DateFieldSetter
extends FieldSetter
{
private final DateWritableV2 value = new DateWritableV2();
public DateFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field)
{
super(rowInspector, row, field);
}
@Override
public void setField(Block block, int position)
{
value.set(toIntExact(DateType.DATE.getLong(block, position)));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class TimestampFieldSetter
extends FieldSetter
{
private final DateTimeZone timeZone;
private final TimestampWritableV2 value = new TimestampWritableV2();
public TimestampFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, DateTimeZone timeZone)
{
super(rowInspector, row, field);
this.timeZone = requireNonNull(timeZone, "timeZone is null");
}
@Override
public void setField(Block block, int position)
{
long epochMilli = TimestampType.TIMESTAMP.getLong(block, position);
epochMilli = timeZone.convertLocalToUTC(epochMilli, false);
value.set(Timestamp.ofEpochMilli(epochMilli));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class DecimalFieldSetter
extends FieldSetter
{
private final HiveDecimalWritable value = new HiveDecimalWritable();
private final DecimalType decimalType;
public DecimalFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, DecimalType decimalType)
{
super(rowInspector, row, field);
this.decimalType = decimalType;
}
@Override
public void setField(Block block, int position)
{
value.set(getHiveDecimal(decimalType, block, position));
rowInspector.setStructFieldData(row, field, value);
}
}
private static class ArrayFieldSetter
extends FieldSetter
{
private final Type elementType;
public ArrayFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type elementType)
{
super(rowInspector, row, field);
this.elementType = requireNonNull(elementType, "elementType is null");
}
@Override
public void setField(Block block, int position)
{
Block arrayBlock = (Block) block.getObject(position, Block.class);
List<Object> list = new ArrayList<>(arrayBlock.getPositionCount());
for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
Object element = HiveWriteUtils.getField(elementType, arrayBlock, i);
list.add(element);
}
rowInspector.setStructFieldData(row, field, list);
}
}
private static class MapFieldSetter
extends FieldSetter
{
private final Type keyType;
private final Type valueType;
public MapFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, Type keyType, Type valueType)
{
super(rowInspector, row, field);
this.keyType = requireNonNull(keyType, "keyType is null");
this.valueType = requireNonNull(valueType, "valueType is null");
}
@Override
public void setField(Block block, int position)
{
Block mapBlock = (Block) block.getObject(position, Block.class);
Map<Object, Object> map = new HashMap<>(mapBlock.getPositionCount() * 2);
for (int i = 0; i < mapBlock.getPositionCount(); i += 2) {
Object key = HiveWriteUtils.getField(keyType, mapBlock, i);
Object value = HiveWriteUtils.getField(valueType, mapBlock, i + 1);
map.put(key, value);
}
rowInspector.setStructFieldData(row, field, map);
}
}
private static class RowFieldSetter
extends FieldSetter
{
private final List<Type> fieldTypes;
public RowFieldSetter(SettableStructObjectInspector rowInspector, Object row, StructField field, List<Type> fieldTypes)
{
super(rowInspector, row, field);
this.fieldTypes = ImmutableList.copyOf(fieldTypes);
}
@Override
public void setField(Block block, int position)
{
Block rowBlock = (Block) block.getObject(position, Block.class);
// TODO reuse row object and use FieldSetters, like we do at the top level
// Ideally, we'd use the same recursive structure starting from the top, but
// this requires modeling row types in the same way we model table rows
// (multiple blocks vs all fields packed in a single block)
List<Object> value = new ArrayList<>(fieldTypes.size());
for (int i = 0; i < fieldTypes.size(); i++) {
Object element = HiveWriteUtils.getField(fieldTypes.get(i), rowBlock, i);
value.add(element);
}
rowInspector.setStructFieldData(row, field, value);
}
}
}

13
presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveBucketingV1.java
View File

@ -109,8 +109,6 @@ public final class HiveBucketingV1
case DATE:
// day offset from 1970-01-01
return toIntExact(prestoType.getLong(block, position));
case TIMESTAMP:
return hashTimestamp(prestoType.getLong(block, position));
default:
throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive primitive category: " + primitiveCategory);
}
@ -161,8 +159,6 @@ public final class HiveBucketingV1
case DATE:
// day offset from 1970-01-01
return toIntExact((long) value);
case TIMESTAMP:
return hashTimestamp((long) value);
default:
throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive primitive category: " + primitiveCategory);
}
@ -176,15 +172,6 @@ public final class HiveBucketingV1
}
}
@SuppressWarnings("NumericCastThatLosesPrecision")
private static int hashTimestamp(long epochMillis)
{
long seconds = (Math.floorDiv(epochMillis, 1000L) << 30);
long nanos = Math.floorMod(epochMillis, 1000) * 1_000_000L;
long secondsAndNanos = seconds | nanos;
return (int) ((secondsAndNanos >>> 32) ^ secondsAndNanos);
}
private static int hashOfMap(MapTypeInfo type, Block singleMapBlock)
{
TypeInfo keyTypeInfo = type.getMapKeyTypeInfo();

25
presto-hive/src/main/java/io/prestosql/plugin/hive/util/SerDeUtils.java
View File

@ -54,12 +54,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspect
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.joda.time.DateTimeZone;
import java.sql.Timestamp;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static com.google.common.base.Preconditions.checkArgument;
import static io.prestosql.spi.type.Chars.truncateToLengthAndTrimSpaces;
@ -73,7 +70,8 @@ public final class SerDeUtils
public static Block getBlockObject(Type type, Object object, ObjectInspector objectInspector)
{
return requireNonNull(serializeObject(type, null, object, objectInspector), "serialized result is null");
Block block = serializeObject(type, null, object, objectInspector);
return requireNonNull(block, "serialized result is null");
}
public static Block serializeObject(Type type, BlockBuilder builder, Object object, ObjectInspector inspector)
@ -272,6 +270,7 @@ public final class SerDeUtils
}
}
@SuppressWarnings("deprecation")
private static long formatDateAsLong(Object object, DateObjectInspector inspector)
{
if (object instanceof LazyDate) {
@ -281,26 +280,14 @@ public final class SerDeUtils
return ((DateWritable) object).getDays();
}
// Hive will return java.sql.Date at midnight in JVM time zone
long millisLocal = inspector.getPrimitiveJavaObject(object).getTime();
// Convert it to midnight in UTC
long millisUtc = DateTimeZone.getDefault().getMillisKeepLocal(DateTimeZone.UTC, millisLocal);
// Convert midnight UTC to days
return TimeUnit.MILLISECONDS.toDays(millisUtc);
return inspector.getPrimitiveJavaObject(object).toEpochDay();
}
private static long formatTimestampAsLong(Object object, TimestampObjectInspector inspector)
{
Timestamp timestamp = getTimestamp(object, inspector);
return timestamp.getTime();
}
private static Timestamp getTimestamp(Object object, TimestampObjectInspector inspector)
{
// handle broken ObjectInspectors
if (object instanceof TimestampWritable) {
return ((TimestampWritable) object).getTimestamp();
return ((TimestampWritable) object).getTimestamp().getTime();
}
return inspector.getPrimitiveJavaObject(object);
return inspector.getPrimitiveJavaObject(object).toEpochMilli();
}
}

15
presto-hive/src/main/java/io/prestosql/plugin/hive/util/Statistics.java
View File

@ -33,7 +33,6 @@ import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.SqlDate;
import io.prestosql.spi.type.SqlDecimal;
import io.prestosql.spi.type.Type;
import org.joda.time.DateTimeZone;
import java.math.BigDecimal;
import java.time.LocalDate;
@ -331,7 +330,6 @@ public final class Statistics
public static Map<String, HiveColumnStatistics> fromComputedStatistics(
ConnectorSession session,
DateTimeZone timeZone,
Map<ColumnStatisticMetadata, Block> computedStatistics,
Map<String, Type> columnTypes,
long rowCount)
@ -344,12 +342,11 @@ public final class Statistics
return result.entrySet()
.stream()
.collect(toImmutableMap(Entry::getKey,
entry -> createHiveColumnStatistics(session, timeZone, entry.getValue(), columnTypes.get(entry.getKey()), rowCount)));
entry -> createHiveColumnStatistics(session, entry.getValue(), columnTypes.get(entry.getKey()), rowCount)));
}
private static HiveColumnStatistics createHiveColumnStatistics(
ConnectorSession session,
DateTimeZone timeZone,
Map<ColumnStatisticType, Block> computedStatistics,
Type columnType,
long rowCount)
@ -360,7 +357,7 @@ public final class Statistics
// We ask the engine to compute either both or neither
verify(computedStatistics.containsKey(MIN_VALUE) == computedStatistics.containsKey(MAX_VALUE));
if (computedStatistics.containsKey(MIN_VALUE)) {
setMinMax(session, timeZone, columnType, computedStatistics.get(MIN_VALUE), computedStatistics.get(MAX_VALUE), result);
setMinMax(session, columnType, computedStatistics.get(MIN_VALUE), computedStatistics.get(MAX_VALUE), result);
}
// MAX_VALUE_SIZE_IN_BYTES
@ -400,7 +397,7 @@ public final class Statistics
return result.build();
}
private static void setMinMax(ConnectorSession session, DateTimeZone timeZone, Type type, Block min, Block max, HiveColumnStatistics.Builder result)
private static void setMinMax(ConnectorSession session, Type type, Block min, Block max, HiveColumnStatistics.Builder result)
{
if (type.equals(BIGINT) || type.equals(INTEGER) || type.equals(SMALLINT) || type.equals(TINYINT)) {
result.setIntegerStatistics(new IntegerStatistics(getIntegerValue(session, type, min), getIntegerValue(session, type, max)));
@ -412,7 +409,7 @@ public final class Statistics
result.setDateStatistics(new DateStatistics(getDateValue(session, type, min), getDateValue(session, type, max)));
}
else if (type.equals(TIMESTAMP)) {
result.setIntegerStatistics(new IntegerStatistics(getTimestampValue(timeZone, min), getTimestampValue(timeZone, max)));
result.setIntegerStatistics(new IntegerStatistics(getTimestampValue(min), getTimestampValue(max)));
}
else if (type instanceof DecimalType) {
result.setDecimalStatistics(new DecimalStatistics(getDecimalValue(session, type, min), getDecimalValue(session, type, max)));
@ -438,10 +435,10 @@ public final class Statistics
return block.isNull(0) ? Optional.empty() : Optional.of(LocalDate.ofEpochDay(((SqlDate) type.getObjectValue(session, block, 0)).getDays()));
}
private static OptionalLong getTimestampValue(DateTimeZone timeZone, Block block)
private static OptionalLong getTimestampValue(Block block)
{
// TODO https://github.com/prestodb/presto/issues/7122
return block.isNull(0) ? OptionalLong.empty() : OptionalLong.of(MILLISECONDS.toSeconds(timeZone.convertUTCToLocal(block.getLong(0, 0))));
return block.isNull(0) ? OptionalLong.empty() : OptionalLong.of(MILLISECONDS.toSeconds(block.getLong(0, 0)));
}
private static Optional<BigDecimal> getDecimalValue(ConnectorSession session, Type type, Block block)

2
presto-hive/src/main/java/io/prestosql/plugin/hive/util/TempFileWriter.java
View File

@ -34,7 +34,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.units.DataSize.Unit.BYTE;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static io.prestosql.orc.metadata.CompressionKind.LZ4;
import static org.joda.time.DateTimeZone.UTC;
public class TempFileWriter
implements Closeable
@ -90,7 +89,6 @@ public class TempFileWriter
.withDictionaryMaxMemory(new DataSize(1, MEGABYTE)),
false,
ImmutableMap.of(),
UTC,
false,
OrcWriteValidationMode.BOTH,
new OrcWriterStats(), Optional.empty(), Optional.empty());

34
presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHive.java
View File

@ -111,13 +111,13 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.TableType;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.IOException;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
@ -129,7 +129,6 @@ import java.util.OptionalDouble;
import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.Set;
import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
@ -235,7 +234,6 @@ import static io.prestosql.spi.type.HyperLogLogType.HYPER_LOG_LOG;
import static io.prestosql.spi.type.IntegerType.INTEGER;
import static io.prestosql.spi.type.RealType.REAL;
import static io.prestosql.spi.type.SmallintType.SMALLINT;
import static io.prestosql.spi.type.TimeZoneKey.UTC_KEY;
import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
import static io.prestosql.spi.type.TinyintType.TINYINT;
import static io.prestosql.spi.type.TypeSignature.parseTypeSignature;
@ -576,8 +574,6 @@ public abstract class AbstractTestHive
protected List<HivePartition> tablePartitionFormatPartitions;
protected List<HivePartition> tableUnpartitionedPartitions;
protected DateTimeZone timeZone;
protected HdfsEnvironment hdfsEnvironment;
protected LocationService locationService;
@ -619,7 +615,7 @@ public abstract class AbstractTestHive
}
}
protected void setupHive(String databaseName, String timeZoneId)
protected void setupHive(String databaseName)
{
database = databaseName;
tablePartitionFormat = new SchemaTableName(database, "presto_test_partition_format");
@ -702,13 +698,13 @@ public abstract class AbstractTestHive
dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 4L)), false)))))),
ImmutableList.of());
tableUnpartitionedProperties = new ConnectorTableProperties();
timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(timeZoneId));
}
protected final void setup(String host, int port, String databaseName, String timeZone)
{
HiveConfig hiveConfig = getHiveConfig();
hiveConfig.setTimeZone(timeZone);
HiveConfig hiveConfig = getHiveConfig()
.setParquetTimeZone(timeZone)
.setRcfileTimeZone(timeZone);
String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
if (proxy != null) {
hiveConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
@ -727,7 +723,7 @@ public abstract class AbstractTestHive
protected final void setup(String databaseName, HiveConfig hiveConfig, HiveMetastore hiveMetastore)
{
setupHive(databaseName, hiveConfig.getTimeZone());
setupHive(databaseName);
metastoreClient = hiveMetastore;
HivePartitionManager partitionManager = new HivePartitionManager(TYPE_MANAGER, hiveConfig);
@ -739,9 +735,7 @@ public abstract class AbstractTestHive
metastoreClient,
hdfsEnvironment,
partitionManager,
timeZone,
10,
true,
false,
false,
false,
@ -768,7 +762,7 @@ public abstract class AbstractTestHive
partitionManager,
new NamenodeStats(),
hdfsEnvironment,
new CachingDirectoryLister(new HiveConfig()),
new CachingDirectoryLister(hiveConfig),
directExecutor(),
new HiveCoercionPolicy(TYPE_MANAGER),
new CounterStat(),
@ -4252,7 +4246,7 @@ public abstract class AbstractTestHive
assertNull(row.getField(index));
}
else {
SqlTimestamp expected = sqlTimestampOf(2011, 5, 6, 7, 8, 9, 123, timeZone, UTC_KEY, SESSION);
SqlTimestamp expected = sqlTimestampOf(2011, 5, 6, 7, 8, 9, 123);
assertEquals(row.getField(index), expected);
}
}
@ -4336,6 +4330,18 @@ public abstract class AbstractTestHive
}
}
// ARRAY<TIMESTAMP>
index = columnIndex.get("t_array_timestamp");
if (index != null) {
if ((rowNumber % 43) == 0) {
assertNull(row.getField(index));
}
else {
SqlTimestamp expected = sqlTimestampOf(LocalDateTime.of(2011, 5, 6, 7, 8, 9, 123_000_000));
assertEquals(row.getField(index), ImmutableList.of(expected));
}
}
// ARRAY<STRUCT<s_string: STRING, s_double:DOUBLE>>
index = columnIndex.get("t_array_struct");
if (index != null) {

30
presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHiveFileFormats.java
View File

@ -43,9 +43,11 @@ import io.prestosql.testing.MaterializedRow;
import io.prestosql.tests.StructuralTestUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.serde2.Serializer;
@ -70,8 +72,6 @@ import java.io.IOException;
import java.lang.invoke.MethodHandle;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@ -96,6 +96,7 @@ import static io.prestosql.plugin.hive.HiveTestUtils.SESSION;
import static io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER;
import static io.prestosql.plugin.hive.HiveTestUtils.isDistinctFrom;
import static io.prestosql.plugin.hive.HiveTestUtils.mapType;
import static io.prestosql.plugin.hive.HiveType.HIVE_TIMESTAMP;
import static io.prestosql.plugin.hive.HiveUtil.isStructuralType;
import static io.prestosql.plugin.hive.util.SerDeUtils.serializeObject;
import static io.prestosql.spi.type.BigintType.BIGINT;
@ -148,17 +149,20 @@ import static org.testng.Assert.assertTrue;
@Test(groups = "hive")
public abstract class AbstractTestHiveFileFormats
{
protected static final DateTimeZone HIVE_STORAGE_TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas");
private static final double EPSILON = 0.001;
private static final long DATE_MILLIS_UTC = new DateTime(2011, 5, 6, 0, 0, UTC).getMillis();
private static final long DATE_DAYS = TimeUnit.MILLISECONDS.toDays(DATE_MILLIS_UTC);
private static final String DATE_STRING = DateTimeFormat.forPattern("yyyy-MM-dd").withZoneUTC().print(
DATE_MILLIS_UTC);
private static final Date SQL_DATE = new Date(UTC.getMillisKeepLocal(DateTimeZone.getDefault(), DATE_MILLIS_UTC));
private static final Date HIVE_DATE = Date.ofEpochMilli(DATE_MILLIS_UTC);
private static final long TIMESTAMP = new DateTime(2011, 5, 6, 7, 8, 9, 123).getMillis();
private static final String TIMESTAMP_STRING = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").print(
private static final String TIMESTAMP_STRING = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS").withZoneUTC().print(
TIMESTAMP);
private static final Timestamp HIVE_TIMESTAMP = Timestamp.ofEpochMilli(TIMESTAMP);
private static final String VARCHAR_MAX_LENGTH_STRING;
@ -297,8 +301,8 @@ public abstract class AbstractTestHiveFileFormats
.add(new TestColumn("t_double", javaDoubleObjectInspector, 6.2, 6.2))
.add(new TestColumn("t_boolean_true", javaBooleanObjectInspector, true, true))
.add(new TestColumn("t_boolean_false", javaBooleanObjectInspector, false, false))
.add(new TestColumn("t_date", javaDateObjectInspector, SQL_DATE, DATE_DAYS))
.add(new TestColumn("t_timestamp", javaTimestampObjectInspector, new Timestamp(TIMESTAMP), TIMESTAMP))
.add(new TestColumn("t_date", javaDateObjectInspector, HIVE_DATE, DATE_DAYS))
.add(new TestColumn("t_timestamp", javaTimestampObjectInspector, HIVE_TIMESTAMP, TIMESTAMP))
.add(new TestColumn("t_decimal_precision_2", DECIMAL_INSPECTOR_PRECISION_2, WRITE_DECIMAL_PRECISION_2,
EXPECTED_DECIMAL_PRECISION_2))
.add(new TestColumn("t_decimal_precision_4", DECIMAL_INSPECTOR_PRECISION_4, WRITE_DECIMAL_PRECISION_4,
@ -359,11 +363,11 @@ public abstract class AbstractTestHiveFileFormats
mapBlockOf(BOOLEAN, BOOLEAN, true, true)))
.add(new TestColumn("t_map_date",
getStandardMapObjectInspector(javaDateObjectInspector, javaDateObjectInspector),
ImmutableMap.of(SQL_DATE, SQL_DATE),
ImmutableMap.of(HIVE_DATE, HIVE_DATE),
mapBlockOf(DateType.DATE, DateType.DATE, DATE_DAYS, DATE_DAYS)))
.add(new TestColumn("t_map_timestamp",
getStandardMapObjectInspector(javaTimestampObjectInspector, javaTimestampObjectInspector),
ImmutableMap.of(new Timestamp(TIMESTAMP), new Timestamp(TIMESTAMP)),
ImmutableMap.of(HIVE_TIMESTAMP, HIVE_TIMESTAMP),
mapBlockOf(TimestampType.TIMESTAMP, TimestampType.TIMESTAMP, TIMESTAMP, TIMESTAMP)))
.add(new TestColumn("t_map_decimal_precision_2",
getStandardMapObjectInspector(DECIMAL_INSPECTOR_PRECISION_2, DECIMAL_INSPECTOR_PRECISION_2),
@ -419,11 +423,11 @@ public abstract class AbstractTestHiveFileFormats
arrayBlockOf(createCharType(10), "test")))
.add(new TestColumn("t_array_date",
getStandardListObjectInspector(javaDateObjectInspector),
ImmutableList.of(SQL_DATE),
ImmutableList.of(HIVE_DATE),
arrayBlockOf(DateType.DATE, DATE_DAYS)))
.add(new TestColumn("t_array_timestamp",
getStandardListObjectInspector(javaTimestampObjectInspector),
ImmutableList.of(new Timestamp(TIMESTAMP)),
ImmutableList.of(HIVE_TIMESTAMP),
StructuralTestUtil.arrayBlockOf(TimestampType.TIMESTAMP, TIMESTAMP)))
.add(new TestColumn("t_array_decimal_precision_2",
getStandardListObjectInspector(DECIMAL_INSPECTOR_PRECISION_2),
@ -561,7 +565,7 @@ public abstract class AbstractTestHiveFileFormats
return columns;
}
public static FileSplit createTestFile(
public static FileSplit createTestFilePresto(
String filePath,
HiveStorageFormat storageFormat,
HiveCompressionCodec compressionCodec,
@ -618,7 +622,7 @@ public abstract class AbstractTestHiveFileFormats
return new FileSplit(new Path(filePath), 0, new File(filePath).length(), new String[0]);
}
public static FileSplit createTestFile(
public static FileSplit createTestFileHive(
String filePath,
HiveStorageFormat storageFormat,
HiveCompressionCodec compressionCodec,
@ -837,7 +841,7 @@ public abstract class AbstractTestHiveFileFormats
assertEquals(actualValue, expectedValue);
}
else if (testColumn.getObjectInspector().getTypeName().equals("timestamp")) {
SqlTimestamp expectedTimestamp = sqlTimestampOf((Long) expectedValue, SESSION);
SqlTimestamp expectedTimestamp = sqlTimestampOf((Long) expectedValue);
assertEquals(actualValue, expectedTimestamp, "Wrong value for column " + testColumn.getName());
}
else if (testColumn.getObjectInspector().getTypeName().startsWith("char")) {

3
presto-hive/src/test/java/io/prestosql/plugin/hive/AbstractTestHiveLocal.java
View File

@ -69,7 +69,8 @@ public abstract class AbstractTestHiveLocal
.build());
HiveConfig hiveConfig = new HiveConfig()
.setTimeZone("America/Los_Angeles");
.setParquetTimeZone("America/Los_Angeles")
.setRcfileTimeZone("America/Los_Angeles");
setup(testDbName, hiveConfig, metastore);
}

3
presto-hive/src/test/java/io/prestosql/plugin/hive/HiveQueryRunner.java
View File

@ -184,7 +184,8 @@ public final class HiveQueryRunner
Map<String, String> hiveProperties = ImmutableMap.<String, String>builder()
.putAll(extraHiveProperties)
.put("hive.time-zone", TIME_ZONE.getID())
.put("hive.rcfile.time-zone", TIME_ZONE.getID())
.put("hive.parquet.time-zone", TIME_ZONE.getID())
.put("hive.security", security)
.put("hive.max-partitions-per-scan", "1000")
.put("hive.assume-canonical-partition-keys", "true")

17
presto-hive/src/test/java/io/prestosql/plugin/hive/HiveTestUtils.java
View File

@ -52,12 +52,15 @@ import io.prestosql.spi.util.BloomFilter;
import io.prestosql.testing.NoOpIndexClient;
import io.prestosql.testing.TestingConnectorSession;
import io.prestosql.type.InternalTypeManager;
import org.apache.hadoop.hive.common.type.Timestamp;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.lang.invoke.MethodHandle;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
@ -94,7 +97,7 @@ public final class HiveTestUtils
FileFormatDataSourceStats stats = new FileFormatDataSourceStats();
HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveConfig);
return ImmutableSet.<HivePageSourceFactory>builder()
.add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats))
.add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats, hiveConfig))
.add(new OrcPageSourceFactory(TYPE_MANAGER, hiveConfig, testHdfsEnvironment, stats, OrcCacheStore.builder().newCacheStore(
new HiveConfig().getOrcFileTailCacheLimit(), Duration.ofMillis(new HiveConfig().getOrcFileTailCacheTtl().toMillis()),
new HiveConfig().getOrcStripeFooterCacheLimit(),
@ -104,10 +107,15 @@ public final class HiveTestUtils
Duration.ofMillis(new HiveConfig().getOrcBloomFiltersCacheTtl().toMillis()),
new HiveConfig().getOrcRowDataCacheMaximumWeight(), Duration.ofMillis(new HiveConfig().getOrcRowDataCacheTtl().toMillis()),
new HiveConfig().isOrcCacheStatsMetricCollectionEnabled())))
.add(new ParquetPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats))
.add(new ParquetPageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats, hiveConfig))
.build();
}
public static HiveRecordCursorProvider createGenericHiveRecordCursorProvider(HdfsEnvironment hdfsEnvironment)
{
return new GenericHiveRecordCursorProvider(hdfsEnvironment);
}
public static Set<HiveSelectivePageSourceFactory> getDefaultHiveSelectiveFactories(HiveConfig hiveConfig)
{
FileFormatDataSourceStats stats = new FileFormatDataSourceStats();
@ -256,4 +264,9 @@ public final class HiveTestUtils
return dynamicFilterSupplier;
}
public static Timestamp hiveTimestamp(LocalDateTime local)
{
return Timestamp.ofEpochSecond(local.toEpochSecond(ZoneOffset.UTC), local.getNano());
}
}

4
presto-hive/src/test/java/io/prestosql/plugin/hive/TestColumnTypeCacheable.java
View File

@ -550,8 +550,8 @@ public class TestColumnTypeCacheable
ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig().setDynamicFilterPartitionFilteringEnabled(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
ColumnMetadata ptdMetadata = new ColumnMetadata("pt_d", TIMESTAMP);
Set<TupleDomain<ColumnMetadata>> cachePredicates = ImmutableSet.of(
TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(TIMESTAMP, HiveUtil.parseHiveTimestamp("1995-10-09 00:00:00", new HiveConfig().getDateTimeZone())))),
TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(TIMESTAMP, HiveUtil.parseHiveTimestamp("1995-11-14 00:00:00", new HiveConfig().getDateTimeZone())))));
TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(TIMESTAMP, HiveUtil.parseHiveTimestamp("1995-10-09 00:00:00")))),
TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(TIMESTAMP, HiveUtil.parseHiveTimestamp("1995-11-14 00:00:00")))));
HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(
session,
"database",

46
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveBucketing.java
View File

@ -22,9 +22,11 @@ import io.prestosql.spi.Page;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.BlockBuilder;
import io.prestosql.spi.type.StandardTypes;
import io.prestosql.spi.type.TimestampType;
import io.prestosql.spi.type.Type;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveVarcharObjectInspector;
@ -32,20 +34,19 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.testng.annotations.Test;
import java.sql.Date;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.prestosql.plugin.hive.HiveBucketing.BucketingVersion.BUCKETING_V1;
import static io.prestosql.plugin.hive.HiveBucketing.BucketingVersion.BUCKETING_V2;
import static io.prestosql.plugin.hive.HiveBucketing.getBucketHashCode;
import static io.prestosql.spi.type.TypeUtils.writeNativeValue;
import static java.lang.Double.longBitsToDouble;
import static java.lang.Float.intBitsToFloat;
import static java.util.Arrays.asList;
import static java.util.Map.Entry;
import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.timestampTypeInfo;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.testng.Assert.assertEquals;
@ -121,14 +122,19 @@ public class TestHiveBucketing
assertBucketEquals("date", Date.valueOf("2015-11-19"), 16758, 8542395);
assertBucketEquals("date", Date.valueOf("1950-11-19"), -6983, -431619185);
assertBucketEquals("timestamp", null, 0, 0);
assertBucketEquals("timestamp", Timestamp.valueOf("1970-01-01 00:00:00.000"), BUCKETING_V1, 7200);
assertBucketEquals("timestamp", Timestamp.valueOf("1969-12-31 23:59:59.999"), BUCKETING_V1, -74736673);
assertBucketEquals("timestamp", Timestamp.valueOf("1950-11-19 12:34:56.789"), BUCKETING_V1, -670699780);
assertBucketEquals("timestamp", Timestamp.valueOf("2015-11-19 07:06:05.432"), BUCKETING_V1, 1278000719);
assertThatThrownBy(() -> assertBucketEquals("timestamp", Timestamp.valueOf("1970-01-01 00:00:00.000"), BUCKETING_V2, 0xDEAD_C0D3))
.isInstanceOf(RuntimeException.class)
.hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
for (BucketingVersion version : BucketingVersion.values()) {
List<TypeInfo> typeInfos = ImmutableList.of(timestampTypeInfo);
assertThatThrownBy(() -> getBucketHashCode(version, typeInfos, new Object[]{0}))
.hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
TimestampType timestampType = TimestampType.TIMESTAMP;
BlockBuilder builder = timestampType.createBlockBuilder(null, 1);
timestampType.writeLong(builder, 0);
Page page = new Page(builder.build());
assertThatThrownBy(() -> getBucketHashCode(version, typeInfos, page, 0))
.hasMessage("Computation of Hive bucket hashCode is not supported for Hive primitive category: TIMESTAMP");
}
assertBucketEquals("array<double>", null, 0, 0);
assertBucketEquals("array<boolean>", ImmutableList.of(), 0, 0);
@ -224,8 +230,8 @@ public class TestHiveBucketing
nativeContainerValues[i] = toNativeContainerValue(type, hiveValue);
}
ImmutableList<Block> blockList = blockListBuilder.build();
int result1 = HiveBucketing.getBucketHashCode(bucketingVersion, hiveTypeInfos, new Page(blockList.toArray(new Block[blockList.size()])), 2);
int result2 = HiveBucketing.getBucketHashCode(bucketingVersion, hiveTypeInfos, nativeContainerValues);
int result1 = getBucketHashCode(bucketingVersion, hiveTypeInfos, new Page(blockList.toArray(new Block[blockList.size()])), 2);
int result2 = getBucketHashCode(bucketingVersion, hiveTypeInfos, nativeContainerValues);
assertEquals(result1, result2, "overloads of getBucketHashCode produced different result");
return result1;
}
@ -316,17 +322,11 @@ public class TestHiveBucketing
case StandardTypes.CHAR:
return Slices.utf8Slice(hiveValue.toString());
case StandardTypes.DATE:
long daysSinceEpochInLocalZone = ((Date) hiveValue).toLocalDate().toEpochDay();
assertEquals(daysSinceEpochInLocalZone, DateWritable.dateToDays((Date) hiveValue));
long daysSinceEpochInLocalZone = ((Date) hiveValue).toEpochDay();
assertEquals(daysSinceEpochInLocalZone, DateWritableV2.dateToDays((Date) hiveValue));
return daysSinceEpochInLocalZone;
case StandardTypes.TIMESTAMP:
Instant instant = ((Timestamp) hiveValue).toInstant();
long epochSecond = instant.getEpochSecond();
int nano = instant.getNano();
assertEquals(nano % 1_000_000, 0);
return epochSecond * 1000 + nano / 1_000_000;
default:
throw new UnsupportedOperationException("unknown type");
throw new IllegalArgumentException("Unsupported bucketing type: " + type);
}
}

13
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveConfig.java
View File

@ -38,7 +38,6 @@ public class TestHiveConfig
public void testDefaults()
{
ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(HiveConfig.class)
.setTimeZone(TimeZone.getDefault().getID())
.setMaxSplitSize(new DataSize(64, Unit.MEGABYTE))
.setMaxPartitionsPerScan(100_000)
.setMaxOutstandingSplits(1_000)
@ -82,6 +81,8 @@ public class TestHiveConfig
.setMaxOpenSortFiles(50)
.setWriteValidationThreads(16)
.setTextMaxLineLength(new DataSize(100, Unit.MEGABYTE))
.setOrcLegacyTimeZone(TimeZone.getDefault().getID())
.setParquetTimeZone(TimeZone.getDefault().getID())
.setUseParquetColumnNames(false)
.setFailOnCorruptedParquetStatistics(true)
.setParquetMaxReadBlockSize(new DataSize(16, Unit.MEGABYTE))
@ -100,6 +101,7 @@ public class TestHiveConfig
.setOrcBloomFiltersCacheEnabled(false).setOrcBloomFiltersCacheTtl(new Duration(4, TimeUnit.HOURS)).setOrcBloomFiltersCacheLimit(250_000)
.setOrcRowDataCacheEnabled(false).setOrcRowDataCacheTtl(new Duration(4, TimeUnit.HOURS)).setOrcRowDataCacheMaximumWeight(new DataSize(20, GIGABYTE))
.setOrcLazyReadSmallRanges(true)
.setRcfileTimeZone(TimeZone.getDefault().getID())
.setRcfileWriterValidate(false)
.setOrcWriteLegacyVersion(false)
.setOrcWriterValidationPercentage(0.0)
@ -153,7 +155,6 @@ public class TestHiveConfig
public void testExplicitPropertyMappings()
{
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
.put("hive.time-zone", nonDefaultTimeZone().getID())
.put("hive.max-split-size", "256MB")
.put("hive.max-partitions-per-scan", "123")
.put("hive.max-outstanding-splits", "10")
@ -199,6 +200,8 @@ public class TestHiveConfig
.put("hive.max-concurrent-file-renames", "100")
.put("hive.assume-canonical-partition-keys", "true")
.put("hive.text.max-line-length", "13MB")
.put("hive.orc.time-zone", nonDefaultTimeZone().getID())
.put("hive.parquet.time-zone", nonDefaultTimeZone().getID())
.put("hive.parquet.use-column-names", "true")
.put("hive.parquet.fail-on-corrupted-statistics", "false")
.put("hive.parquet.max-read-block-size", "66kB")
@ -226,6 +229,7 @@ public class TestHiveConfig
.put("hive.orc.row-data.block.cache.ttl", "1h")
.put("hive.orc.row-data.block.cache.max.weight", "1MB")
.put("hive.orc.lazy-read-small-ranges", "false")
.put("hive.rcfile.time-zone", nonDefaultTimeZone().getID())
.put("hive.rcfile.writer.validate", "true")
.put("hive.orc.writer.use-legacy-version-number", "true")
.put("hive.orc.writer.validation-percentage", "0.16")
@ -275,7 +279,6 @@ public class TestHiveConfig
.build();
HiveConfig expected = new HiveConfig()
.setTimeZone(nonDefaultTimeZone().toTimeZone().getID())
.setMaxSplitSize(new DataSize(256, Unit.MEGABYTE))
.setMaxPartitionsPerScan(123)
.setMaxOutstandingSplits(10)
@ -318,11 +321,14 @@ public class TestHiveConfig
.setDomainSocketPath("/foo")
.setS3FileSystemType(S3FileSystemType.EMRFS)
.setTextMaxLineLength(new DataSize(13, Unit.MEGABYTE))
.setOrcLegacyTimeZone(nonDefaultTimeZone().getID())
.setParquetTimeZone(nonDefaultTimeZone().getID())
.setUseParquetColumnNames(true)
.setFailOnCorruptedParquetStatistics(false)
.setParquetMaxReadBlockSize(new DataSize(66, Unit.KILOBYTE))
.setUseOrcColumnNames(true)
.setAssumeCanonicalPartitionKeys(true)
.setRcfileTimeZone(nonDefaultTimeZone().getID())
.setOrcBloomFiltersEnabled(true)
.setOrcDefaultBloomFilterFpp(0.96)
.setOrcMaxMergeDistance(new DataSize(22, Unit.KILOBYTE))
@ -336,6 +342,7 @@ public class TestHiveConfig
.setOrcBloomFiltersCacheEnabled(true).setOrcBloomFiltersCacheTtl(new Duration(1, TimeUnit.HOURS)).setOrcBloomFiltersCacheLimit(100)
.setOrcRowDataCacheEnabled(true).setOrcRowDataCacheTtl(new Duration(1, TimeUnit.HOURS)).setOrcRowDataCacheMaximumWeight(new DataSize(1, MEGABYTE))
.setOrcLazyReadSmallRanges(false)
.setRcfileTimeZone(nonDefaultTimeZone().getID())
.setRcfileWriterValidate(true)
.setOrcWriteLegacyVersion(true)
.setOrcWriterValidationPercentage(0.16)

150
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java
View File

@ -44,7 +44,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.apache.hadoop.mapred.FileSplit;
import org.joda.time.DateTimeZone;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@ -73,6 +72,9 @@ import static io.prestosql.plugin.hive.HiveStorageFormat.RCBINARY;
import static io.prestosql.plugin.hive.HiveStorageFormat.RCTEXT;
import static io.prestosql.plugin.hive.HiveStorageFormat.SEQUENCEFILE;
import static io.prestosql.plugin.hive.HiveStorageFormat.TEXTFILE;
import static io.prestosql.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER;
import static io.prestosql.plugin.hive.HiveTestUtils.createGenericHiveRecordCursorProvider;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT;
@ -91,8 +93,6 @@ public class TestHiveFileFormats
private static TestingConnectorSession parquetPageSourceSession = new TestingConnectorSession(new HiveSessionProperties(createParquetHiveConfig(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
private static TestingConnectorSession parquetPageSourceSessionUseName = new TestingConnectorSession(new HiveSessionProperties(createParquetHiveConfig(true), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
private static final DateTimeZone HIVE_STORAGE_TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas");
@DataProvider(name = "rowCount")
public static Object[][] rowCountProvider()
{
@ -119,7 +119,7 @@ public class TestHiveFileFormats
assertThatFileFormat(TEXTFILE)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test(dataProvider = "rowCount")
@ -133,7 +133,7 @@ public class TestHiveFileFormats
assertThatFileFormat(SEQUENCEFILE)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test(dataProvider = "rowCount")
@ -150,7 +150,7 @@ public class TestHiveFileFormats
assertThatFileFormat(CSV)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test
@ -162,7 +162,7 @@ public class TestHiveFileFormats
new TestColumn("t_null_string", javaStringObjectInspector, null, Slices.utf8Slice("")), // null was converted to empty string!
new TestColumn("t_string", javaStringObjectInspector, "test", Slices.utf8Slice("test"))))
.withRowsCount(2)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test(dataProvider = "rowCount")
@ -180,9 +180,7 @@ public class TestHiveFileFormats
.filter(column -> !column.getName().equals("t_map_float"))
.filter(column -> !column.getName().equals("t_map_double"))
// null map keys are not supported
.filter(column -> !column.getName().equals("t_map_null_key"))
.filter(column -> !column.getName().equals("t_map_null_key_complex_key_value"))
.filter(column -> !column.getName().equals("t_map_null_key_complex_value"))
.filter(TestHiveFileFormats::withoutNullMapKeyTests)
// decimal(38) is broken or not supported
.filter(column -> !column.getName().equals("t_decimal_precision_38"))
.filter(column -> !column.getName().equals("t_map_decimal_precision_38"))
@ -192,7 +190,7 @@ public class TestHiveFileFormats
assertThatFileFormat(JSON)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test(dataProvider = "rowCount")
@ -208,7 +206,7 @@ public class TestHiveFileFormats
assertThatFileFormat(RCTEXT)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test(dataProvider = "rowCount")
@ -218,7 +216,7 @@ public class TestHiveFileFormats
assertThatFileFormat(RCTEXT)
.withColumns(TEST_COLUMNS)
.withRowsCount(rowCount)
.isReadableByPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
}
@Test(dataProvider = "rowCount")
@ -233,25 +231,9 @@ public class TestHiveFileFormats
assertThatFileFormat(RCTEXT)
.withColumns(testColumns)
.withRowsCount(rowCount)
.withFileWriterFactory(new RcFileFileWriterFactory(HiveTestUtils.HDFS_ENVIRONMENT, HiveTestUtils.TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT))
.isReadableByPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
}
@Test(dataProvider = "rowCount")
public void testRCBinary(int rowCount)
throws Exception
{
// RCBinary does not support complex type as key of a map and interprets empty VARCHAR as nulls
List<TestColumn> testColumns = TEST_COLUMNS.stream()
.filter(testColumn -> {
String name = testColumn.getName();
return !name.equals("t_map_null_key_complex_key_value") && !name.equals("t_empty_varchar");
}).collect(toList());
assertThatFileFormat(RCBINARY)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.withFileWriterFactory(new RcFileFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT))
.isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
}
@Test(dataProvider = "rowCount")
@ -259,14 +241,16 @@ public class TestHiveFileFormats
throws Exception
{
// RCBinary does not support complex type as key of a map and interprets empty VARCHAR as nulls
// Hive binary writers are broken for timestamps
List<TestColumn> testColumns = TEST_COLUMNS.stream()
.filter(testColumn -> !testColumn.getName().equals("t_empty_varchar"))
.filter(TestHiveFileFormats::withoutTimestamps)
.collect(toList());
assertThatFileFormat(RCBINARY)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
}
@Test(dataProvider = "rowCount")
@ -280,22 +264,33 @@ public class TestHiveFileFormats
.filter(TestHiveFileFormats::withoutNullMapKeyTests)
.collect(toList());
// Hive cannot read timestamps from old files
List<TestColumn> testColumnsNoTimestamps = testColumns.stream()
.filter(TestHiveFileFormats::withoutTimestamps)
.collect(toList());
assertThatFileFormat(RCBINARY)
.withColumns(testColumns)
.withRowsCount(rowCount)
.withFileWriterFactory(new RcFileFileWriterFactory(HiveTestUtils.HDFS_ENVIRONMENT, HiveTestUtils.TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT))
.isReadableByPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.withFileWriterFactory(new RcFileFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS))
.isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()))
.withColumns(testColumnsNoTimestamps)
.isReadableByRecordCursor(createGenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test(dataProvider = "rowCount")
public void testOrc(int rowCount)
throws Exception
{
// Hive binary writers are broken for timestamps
List<TestColumn> testColumns = TEST_COLUMNS.stream()
.filter(TestHiveFileFormats::withoutTimestamps)
.collect(toImmutableList());
assertThatFileFormat(ORC)
.withColumns(TEST_COLUMNS)
.withColumns(testColumns)
.withRowsCount(rowCount)
.isReadableByPageSource(new OrcPageSourceFactory(HiveTestUtils.TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HiveTestUtils.HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
.isReadableByPageSource(new OrcPageSourceFactory(TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
new HiveConfig().getOrcFileTailCacheLimit(), Duration.ofMillis(new HiveConfig().getOrcFileTailCacheTtl().toMillis()),
new HiveConfig().getOrcStripeFooterCacheLimit(),
Duration.ofMillis(new HiveConfig().getOrcStripeFooterCacheTtl().toMillis()),
@ -319,16 +314,16 @@ public class TestHiveFileFormats
// A Presto page can not contain a map with null keys, so a page based writer can not write null keys
List<TestColumn> testColumns = TEST_COLUMNS.stream()
.filter(testColumn -> !testColumn.getName().equals("t_map_null_key") && !testColumn.getName().equals("t_map_null_key_complex_value") && !testColumn.getName().equals("t_map_null_key_complex_key_value"))
.filter(TestHiveFileFormats::withoutNullMapKeyTests)
.collect(toList());
assertThatFileFormat(ORC)
.withColumns(testColumns)
.withRowsCount(rowCount)
.withSession(session)
.withFileWriterFactory(new OrcFileWriterFactory(HiveTestUtils.HDFS_ENVIRONMENT, HiveTestUtils.TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, false, STATS, new OrcWriterOptions()))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT))
.isReadableByPageSource(new OrcPageSourceFactory(HiveTestUtils.TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HiveTestUtils.HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
.withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, TYPE_MANAGER, new NodeVersion("test"), false, STATS, new OrcWriterOptions()))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT))
.isReadableByPageSource(new OrcPageSourceFactory(TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
new HiveConfig().getOrcFileTailCacheLimit(), Duration.ofMillis(new HiveConfig().getOrcFileTailCacheTtl().toMillis()),
new HiveConfig().getOrcStripeFooterCacheLimit(),
Duration.ofMillis(new HiveConfig().getOrcStripeFooterCacheTtl().toMillis()),
@ -345,12 +340,17 @@ public class TestHiveFileFormats
{
TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig(), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
// Hive binary writers are broken for timestamps
List<TestColumn> testColumns = TEST_COLUMNS.stream()
.filter(TestHiveFileFormats::withoutTimestamps)
.collect(toImmutableList());
assertThatFileFormat(ORC)
.withWriteColumns(TEST_COLUMNS)
.withWriteColumns(testColumns)
.withRowsCount(rowCount)
.withReadColumns(Lists.reverse(TEST_COLUMNS))
.withReadColumns(Lists.reverse(testColumns))
.withSession(session)
.isReadableByPageSource(new OrcPageSourceFactory(HiveTestUtils.TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(true), HiveTestUtils.HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
.isReadableByPageSource(new OrcPageSourceFactory(TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(true), HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
new HiveConfig().getOrcFileTailCacheLimit(), Duration.ofMillis(new HiveConfig().getOrcFileTailCacheTtl().toMillis()),
new HiveConfig().getOrcStripeFooterCacheLimit(),
Duration.ofMillis(new HiveConfig().getOrcStripeFooterCacheTtl().toMillis()),
@ -385,7 +385,7 @@ public class TestHiveFileFormats
assertThatFileFormat(AVRO)
.withColumns(getTestColumnsSupportedByAvro())
.withRowsCount(rowCount)
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
private static List<TestColumn> getTestColumnsSupportedByAvro()
@ -407,7 +407,7 @@ public class TestHiveFileFormats
.withColumns(testColumns)
.withSession(parquetPageSourceSession)
.withRowsCount(rowCount)
.isReadableByPageSource(new ParquetPageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
}
@Test(dataProvider = "rowCount")
@ -430,7 +430,7 @@ public class TestHiveFileFormats
.withReadColumns(readColumns)
.withSession(parquetPageSourceSession)
.withRowsCount(rowCount)
.isReadableByPageSource(new ParquetPageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
// test name-based access
readColumns = Lists.reverse(writeColumns);
@ -438,7 +438,7 @@ public class TestHiveFileFormats
.withWriteColumns(writeColumns)
.withReadColumns(readColumns)
.withSession(parquetPageSourceSessionUseName)
.isReadableByPageSource(new ParquetPageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
}
private static List<TestColumn> getTestColumnsSupportedByParquet()
@ -466,19 +466,19 @@ public class TestHiveFileFormats
assertThatFileFormat(RCTEXT)
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.isReadableByPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(RCBINARY)
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.isReadableByPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(ORC)
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.isReadableByPageSource(new OrcPageSourceFactory(HiveTestUtils.TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HiveTestUtils.HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
.isReadableByPageSource(new OrcPageSourceFactory(TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
new HiveConfig().getOrcFileTailCacheLimit(), Duration.ofMillis(new HiveConfig().getOrcFileTailCacheTtl().toMillis()),
new HiveConfig().getOrcStripeFooterCacheLimit(),
Duration.ofMillis(new HiveConfig().getOrcStripeFooterCacheTtl().toMillis()),
@ -492,22 +492,22 @@ public class TestHiveFileFormats
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.withSession(parquetPageSourceSession)
.isReadableByPageSource(new ParquetPageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS));
.isReadableByPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()));
assertThatFileFormat(AVRO)
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(SEQUENCEFILE)
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
assertThatFileFormat(TEXTFILE)
.withWriteColumns(ImmutableList.of(writeColumn))
.withReadColumns(ImmutableList.of(readColumn))
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT));
.isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT));
}
@Test
@ -524,17 +524,17 @@ public class TestHiveFileFormats
assertThatFileFormat(RCTEXT)
.withColumns(columns)
.isFailingForPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage)
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
.isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()), expectedErrorCode, expectedMessage)
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
assertThatFileFormat(RCBINARY)
.withColumns(columns)
.isFailingForPageSource(new RcFilePageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage)
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
.isFailingForPageSource(new RcFilePageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()), expectedErrorCode, expectedMessage)
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
assertThatFileFormat(ORC)
.withColumns(columns)
.isFailingForPageSource(new OrcPageSourceFactory(HiveTestUtils.TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HiveTestUtils.HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
.isFailingForPageSource(new OrcPageSourceFactory(TYPE_MANAGER, new HiveConfig().setUseOrcColumnNames(false), HDFS_ENVIRONMENT, STATS, OrcCacheStore.builder().newCacheStore(
new HiveConfig().getOrcFileTailCacheLimit(), Duration.ofMillis(new HiveConfig().getOrcFileTailCacheTtl().toMillis()),
new HiveConfig().getOrcStripeFooterCacheLimit(),
Duration.ofMillis(new HiveConfig().getOrcStripeFooterCacheTtl().toMillis()),
@ -547,15 +547,15 @@ public class TestHiveFileFormats
assertThatFileFormat(PARQUET)
.withColumns(columns)
.withSession(parquetPageSourceSession)
.isFailingForPageSource(new ParquetPageSourceFactory(HiveTestUtils.TYPE_MANAGER, HiveTestUtils.HDFS_ENVIRONMENT, STATS), expectedErrorCode, expectedMessage);
.isFailingForPageSource(new ParquetPageSourceFactory(TYPE_MANAGER, HDFS_ENVIRONMENT, STATS, new HiveConfig()), expectedErrorCode, expectedMessage);
assertThatFileFormat(SEQUENCEFILE)
.withColumns(columns)
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
assertThatFileFormat(TEXTFILE)
.withColumns(columns)
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HiveTestUtils.HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
.isFailingForRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT), expectedErrorCode, expectedMessage);
}
private void testCursorProvider(HiveRecordCursorProvider cursorProvider,
@ -592,8 +592,7 @@ public class TestHiveFileFormats
TupleDomain.all(),
getColumnHandles(testColumns),
partitionKeys,
DateTimeZone.getDefault(),
HiveTestUtils.TYPE_MANAGER,
TYPE_MANAGER,
ImmutableMap.of(),
Optional.empty(),
false,
@ -646,8 +645,7 @@ public class TestHiveFileFormats
TupleDomain.all(),
columnHandles,
partitionKeys,
DateTimeZone.getDefault(),
HiveTestUtils.TYPE_MANAGER,
TYPE_MANAGER,
ImmutableMap.of(),
Optional.empty(),
false,
@ -835,10 +833,10 @@ public class TestHiveFileFormats
try {
FileSplit split;
if (fileWriterFactory != null) {
split = createTestFile(file.getAbsolutePath(), storageFormat, compressionCodec, writeColumns, session, rowsCount, fileWriterFactory);
split = createTestFilePresto(file.getAbsolutePath(), storageFormat, compressionCodec, writeColumns, session, rowsCount, fileWriterFactory);
}
else {
split = createTestFile(file.getAbsolutePath(), storageFormat, compressionCodec, writeColumns, rowsCount);
split = createTestFileHive(file.getAbsolutePath(), storageFormat, compressionCodec, writeColumns, rowsCount);
}
if (pageSourceFactory.isPresent()) {
testPageSourceFactory(pageSourceFactory.get(), split, storageFormat, readColumns, session, rowsCount);
@ -870,4 +868,12 @@ public class TestHiveFileFormats
}
}
}
private static boolean withoutTimestamps(TestColumn testColumn)
{
String name = testColumn.getName();
return !name.equals("t_timestamp") &&
!name.equals("t_map_timestamp") &&
!name.equals("t_array_timestamp");
}
}

4
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveUtil.java
View File

@ -100,7 +100,7 @@ public class TestHiveUtil
@Test
public void testParseHiveTimestamp()
{
DateTime time = new DateTime(2011, 5, 6, 7, 8, 9, 123, nonDefaultTimeZone());
DateTime time = new DateTime(2011, 5, 6, 7, 8, 9, 123, DateTimeZone.UTC);
assertEquals(parse(time, "yyyy-MM-dd HH:mm:ss"), unixTime(time, 0));
assertEquals(parse(time, "yyyy-MM-dd HH:mm:ss.S"), unixTime(time, 1));
assertEquals(parse(time, "yyyy-MM-dd HH:mm:ss.SSS"), unixTime(time, 3));
@ -232,7 +232,7 @@ public class TestHiveUtil
private static long parse(DateTime time, String pattern)
{
return parseHiveTimestamp(DateTimeFormat.forPattern(pattern).print(time), nonDefaultTimeZone());
return parseHiveTimestamp(DateTimeFormat.forPattern(pattern).print(time));
}
private static long unixTime(DateTime time, int factionalDigits)

2
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveWriterFactory.java
View File

@ -60,6 +60,7 @@ import static io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveFileWriterFac
import static io.prestosql.plugin.hive.HiveTestUtils.getDefaultOrcFileWriterFactory;
import static java.util.concurrent.Executors.newCachedThreadPool;
import static org.apache.hadoop.hive.ql.exec.Utilities.getBucketIdFromFile;
import static org.joda.time.DateTimeZone.UTC;
import static org.testng.Assert.assertEquals;
public class TestHiveWriterFactory
@ -140,6 +141,7 @@ public class TestHiveWriterFactory
hiveConfig.getWriterSortBufferSize(),
hiveConfig.getMaxOpenSortFiles(),
false,
UTC,
session,
new TestingNodeManager("fake-environment"),
new HiveEventClient(),

2
presto-hive/src/test/java/io/prestosql/plugin/hive/TestOrcPageSourceMemoryTracking.java
View File

@ -74,7 +74,6 @@ import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.orc.NullMemoryManager;
import org.apache.orc.impl.WriterImpl;
import org.joda.time.DateTimeZone;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
@ -507,7 +506,6 @@ public class TestOrcPageSourceMemoryTracking
TupleDomain.all(),
columns,
partitionKeys,
DateTimeZone.UTC,
TYPE_MANAGER,
ImmutableMap.of(),
Optional.empty(),

21
presto-hive/src/test/java/io/prestosql/plugin/hive/benchmark/FileFormat.java
View File

@ -55,7 +55,6 @@ import io.prestosql.spi.type.Type;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.joda.time.DateTimeZone;
import java.io.File;
import java.io.FileOutputStream;
@ -77,6 +76,7 @@ import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB;
import static org.joda.time.DateTimeZone.UTC;
public enum FileFormat
{
@ -84,7 +84,7 @@ public enum FileFormat
@Override
public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List<String> columnNames, List<Type> columnTypes)
{
HivePageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats());
HivePageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HiveConfig().setRcfileTimeZone("UTC"));
return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.RCBINARY);
}
@ -100,7 +100,7 @@ public enum FileFormat
return new PrestoRcFileFormatWriter(
targetFile,
columnTypes,
new BinaryRcFileEncoding(),
new BinaryRcFileEncoding(UTC),
compressionCodec);
}
},
@ -108,7 +108,7 @@ public enum FileFormat
@Override
public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List<String> columnNames, List<Type> columnTypes)
{
HivePageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats());
HivePageSourceFactory pageSourceFactory = new RcFilePageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HiveConfig().setRcfileTimeZone("UTC"));
return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.RCTEXT);
}
@ -124,7 +124,7 @@ public enum FileFormat
return new PrestoRcFileFormatWriter(
targetFile,
columnTypes,
new TextRcFileEncoding(DateTimeZone.forID(session.getTimeZoneKey().getId())),
new TextRcFileEncoding(),
compressionCodec);
}
},
@ -158,7 +158,6 @@ public enum FileFormat
targetFile,
columnNames,
columnTypes,
DateTimeZone.forID(session.getTimeZoneKey().getId()),
compressionCodec);
}
},
@ -167,7 +166,7 @@ public enum FileFormat
@Override
public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List<String> columnNames, List<Type> columnTypes)
{
HivePageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats());
HivePageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HiveConfig());
return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.PARQUET);
}
@ -247,7 +246,7 @@ public enum FileFormat
@Override
public ConnectorPageSource createFileFormatReader(ConnectorSession session, HdfsEnvironment hdfsEnvironment, File targetFile, List<String> columnNames, List<Type> columnTypes)
{
HivePageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats());
HivePageSourceFactory pageSourceFactory = new ParquetPageSourceFactory(TYPE_MANAGER, hdfsEnvironment, new FileFormatDataSourceStats(), new HiveConfig());
return createPageSource(pageSourceFactory, session, targetFile, columnNames, columnTypes, HiveStorageFormat.PARQUET);
}
@ -316,7 +315,6 @@ public enum FileFormat
createSchema(format, columnNames, columnTypes),
columnHandles,
TupleDomain.all(),
DateTimeZone.forID(session.getTimeZoneKey().getId()),
TYPE_MANAGER,
false,
ImmutableMap.of())
@ -352,7 +350,6 @@ public enum FileFormat
createSchema(format, columnNames, columnTypes),
columnHandles,
TupleDomain.all(),
DateTimeZone.forID(session.getTimeZoneKey().getId()),
Optional.empty(),
Optional.empty(),
Optional.empty(),
@ -386,6 +383,7 @@ public enum FileFormat
format.getEstimatedWriterSystemMemoryUsage(),
config,
TYPE_MANAGER,
UTC,
session);
}
@ -458,7 +456,7 @@ public enum FileFormat
{
private final OrcWriter writer;
public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Type> types, DateTimeZone hiveStorageTimeZone, HiveCompressionCodec compressionCodec)
public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Type> types, HiveCompressionCodec compressionCodec)
throws IOException
{
writer = new OrcWriter(
@ -469,7 +467,6 @@ public enum FileFormat
new OrcWriterOptions(),
false,
ImmutableMap.of(),
hiveStorageTimeZone,
false,
BOTH,
new OrcWriterStats(), Optional.empty(), Optional.empty());

2
presto-hive/src/test/java/io/prestosql/plugin/hive/orc/TestOrcAcidPageSource.java
View File

@ -38,7 +38,6 @@ import io.prestosql.type.InternalTypeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.joda.time.DateTimeZone;
import org.testng.annotations.Test;
import java.io.File;
@ -176,7 +175,6 @@ public class TestOrcAcidPageSource
createSchema(),
columnHandles,
tupleDomain,
DateTimeZone.UTC,
Optional.empty(),
deleteDeltaLocations,
Optional.empty(),

16
presto-hive/src/test/java/io/prestosql/plugin/hive/parquet/AbstractTestParquetReader.java
View File

@ -28,7 +28,9 @@ import io.prestosql.spi.type.SqlDecimal;
import io.prestosql.spi.type.SqlTimestamp;
import io.prestosql.spi.type.SqlVarbinary;
import io.prestosql.spi.type.Type;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
@ -39,8 +41,6 @@ import org.testng.annotations.Test;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
@ -64,7 +64,6 @@ import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.cycle;
import static com.google.common.collect.Iterables.limit;
import static com.google.common.collect.Iterables.transform;
import static io.prestosql.plugin.hive.parquet.ParquetTester.HIVE_STORAGE_TIME_ZONE;
import static io.prestosql.plugin.hive.parquet.ParquetTester.insertNullEvery;
import static io.prestosql.spi.type.BigintType.BIGINT;
import static io.prestosql.spi.type.BooleanType.BOOLEAN;
@ -80,7 +79,6 @@ import static io.prestosql.spi.type.VarbinaryType.VARBINARY;
import static io.prestosql.spi.type.VarcharType.VARCHAR;
import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType;
import static io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf;
import static io.prestosql.testing.TestingConnectorSession.SESSION;
import static io.prestosql.tests.StructuralTestUtil.mapType;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
@ -122,7 +120,7 @@ public abstract class AbstractTestParquetReader
@BeforeClass
public void setUp()
{
assertEquals(DateTimeZone.getDefault(), HIVE_STORAGE_TIME_ZONE);
assertEquals(DateTimeZone.getDefault(), DateTimeZone.forID("America/Bahia_Banderas"));
// Parquet has excessive logging at INFO level
parquetLogger = Logger.getLogger("org.apache.parquet.hadoop");
@ -1774,7 +1772,7 @@ public abstract class AbstractTestParquetReader
if (input == null) {
return null;
}
Timestamp timestamp = new Timestamp(0);
Timestamp timestamp = new Timestamp();
long seconds = (input / 1000);
int nanos = ((input % 1000) * 1_000_000);
@ -1789,7 +1787,7 @@ public abstract class AbstractTestParquetReader
nanos -= 1_000_000_000;
seconds += 1;
}
timestamp.setTime(seconds * 1000);
timestamp.setTimeInMillis(seconds * 1000);
timestamp.setNanos(nanos);
return timestamp;
}
@ -1799,7 +1797,7 @@ public abstract class AbstractTestParquetReader
if (input == null) {
return null;
}
return sqlTimestampOf(input, SESSION);
return sqlTimestampOf(input);
}
private static Date intToDate(Integer input)
@ -1807,7 +1805,7 @@ public abstract class AbstractTestParquetReader
if (input == null) {
return null;
}
return Date.valueOf(LocalDate.ofEpochDay(input));
return Date.valueOf(LocalDate.ofEpochDay(input).toString());
}
private static SqlDate intToSqlDate(Integer input)

5
presto-hive/src/test/java/io/prestosql/plugin/hive/parquet/ParquetTester.java
View File

@ -64,7 +64,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.schema.MessageType;
import org.joda.time.DateTimeZone;
import java.io.Closeable;
import java.io.File;
@ -90,7 +89,6 @@ import static io.prestosql.plugin.hive.HiveUtil.isArrayType;
import static io.prestosql.plugin.hive.HiveUtil.isMapType;
import static io.prestosql.plugin.hive.HiveUtil.isRowType;
import static io.prestosql.plugin.hive.HiveUtil.isStructuralType;
import static io.prestosql.spi.type.TimeZoneKey.UTC_KEY;
import static io.prestosql.spi.type.VarbinaryType.VARBINARY;
import static io.prestosql.spi.type.Varchars.isVarcharType;
import static java.util.Arrays.stream;
@ -113,7 +111,6 @@ import static org.testng.Assert.assertTrue;
public class ParquetTester
{
public static final DateTimeZone HIVE_STORAGE_TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas");
private static final boolean OPTIMIZED = true;
private static final HiveConfig HIVE_CLIENT_CONFIG = createHiveConfig(false);
private static final HdfsEnvironment HDFS_ENVIRONMENT = HiveTestUtils.createTestHdfsEnvironment(HIVE_CLIENT_CONFIG);
@ -474,7 +471,7 @@ public class ParquetTester
return new SqlDate(((Long) fieldFromCursor).intValue());
}
if (TimestampType.TIMESTAMP.equals(type)) {
return new SqlTimestamp((long) fieldFromCursor, UTC_KEY);
return new SqlTimestamp((long) fieldFromCursor);
}
return fieldFromCursor;
}

6
presto-hive/src/test/java/io/prestosql/plugin/hive/parquet/TestParquetPageSourceFactory.java
View File

@ -51,7 +51,7 @@ public class TestParquetPageSourceFactory
{
HiveHdfsConfiguration hiveHdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveConfig(), ImmutableSet.of()), ImmutableSet.of());
HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hiveHdfsConfiguration, new HiveConfig(), new NoHdfsAuthentication());
parquetPageSourceFactory = new ParquetPageSourceFactory(new TestingTypeManager(), hdfsEnvironment, new FileFormatDataSourceStats());
parquetPageSourceFactory = new ParquetPageSourceFactory(new TestingTypeManager(), hdfsEnvironment, new FileFormatDataSourceStats(), new HiveConfig());
}
@AfterClass(alwaysRun = true)
@ -68,7 +68,7 @@ public class TestParquetPageSourceFactory
schema.setProperty(SERIALIZATION_LIB, "");
schema.setProperty(FILE_INPUT_FORMAT, "");
schema.setProperty(FILE_OUTPUT_FORMAT, "");
Optional<? extends ConnectorPageSource> optionalPageSource = parquetPageSourceFactory.createPageSource(new Configuration(), null, null, 0L, 0L, 0L, schema, null, null, null, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L);
Optional<? extends ConnectorPageSource> optionalPageSource = parquetPageSourceFactory.createPageSource(new Configuration(), null, null, 0L, 0L, 0L, schema, null, null, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L);
assertFalse(optionalPageSource.isPresent());
}
@ -80,7 +80,7 @@ public class TestParquetPageSourceFactory
schema.setProperty(SERIALIZATION_LIB, PARQUET.getSerDe());
schema.setProperty(FILE_INPUT_FORMAT, HoodieParquetRealtimeInputFormat.class.getName());
schema.setProperty(FILE_OUTPUT_FORMAT, "");
Optional<? extends ConnectorPageSource> optionalPageSource = parquetPageSourceFactory.createPageSource(new Configuration(), null, null, 0L, 0L, 0L, schema, null, null, null, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L);
Optional<? extends ConnectorPageSource> optionalPageSource = parquetPageSourceFactory.createPageSource(new Configuration(), null, null, 0L, 0L, 0L, schema, null, null, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), null, false, -1L);
assertTrue(shouldUseRecordReaderFromInputFormat(new Configuration(), schema));
assertFalse(optionalPageSource.isPresent());
}

7
presto-hive/src/test/java/io/prestosql/plugin/hive/parquet/write/TestDataWritableWriter.java
View File

@ -14,11 +14,12 @@
package io.prestosql.plugin.hive.parquet.write;
import io.airlift.log.Logger;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils;
import org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriter;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.ParquetHiveRecord;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
@ -46,8 +47,6 @@ import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.OriginalType;
import org.apache.parquet.schema.Type;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.List;
import java.util.Map;
@ -377,7 +376,7 @@ public class TestDataWritableWriter
break;
case DATE:
Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value);
recordConsumer.addInteger(DateWritable.dateToDays(vDate));
recordConsumer.addInteger(vDate.toEpochDay());
break;
default:
throw new IllegalArgumentException("Unsupported primitive data type: " + inspector.getPrimitiveCategory());

5
presto-hive/src/test/java/io/prestosql/plugin/hive/statistics/TestMetastoreHiveStatisticsProvider.java
View File

@ -42,7 +42,6 @@ import io.prestosql.spi.statistics.TableStatistics;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.Type;
import io.prestosql.testing.TestingConnectorSession;
import org.joda.time.DateTimeZone;
import org.testng.annotations.Test;
import java.math.BigDecimal;
@ -474,7 +473,7 @@ public class TestMetastoreHiveStatisticsProvider
private static void assertConvertPartitionValueToDouble(Type type, String value, double expected)
{
Object prestoValue = parsePartitionValue(format("p=%s", value), value, type, DateTimeZone.getDefault()).getValue();
Object prestoValue = parsePartitionValue(format("p=%s", value), value, type).getValue();
assertEquals(convertPartitionValueToDouble(type, prestoValue), expected);
}
@ -777,7 +776,7 @@ public class TestMetastoreHiveStatisticsProvider
private static HivePartition partition(String name)
{
return parsePartition(TABLE, name, ImmutableList.of(PARTITION_COLUMN_1, PARTITION_COLUMN_2), ImmutableList.of(VARCHAR, BIGINT), DateTimeZone.getDefault());
return parsePartition(TABLE, name, ImmutableList.of(PARTITION_COLUMN_1, PARTITION_COLUMN_2), ImmutableList.of(VARCHAR, BIGINT));
}
private static PartitionStatistics rowsCount(long rowsCount)

13
presto-hive/src/test/java/io/prestosql/plugin/hive/util/TestSerDeUtils.java
View File

@ -28,6 +28,8 @@ import io.prestosql.spi.block.BlockEncodingSerde;
import io.prestosql.spi.type.ArrayType;
import io.prestosql.spi.type.RowType;
import io.prestosql.tests.StructuralTestUtil;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.io.BytesWritable;
@ -35,7 +37,7 @@ import org.joda.time.DateTime;
import org.testng.annotations.Test;
import java.lang.reflect.Type;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -58,6 +60,7 @@ import static io.prestosql.tests.StructuralTestUtil.mapBlockOf;
import static io.prestosql.tests.StructuralTestUtil.rowBlockOf;
import static java.lang.Double.doubleToLongBits;
import static java.lang.Float.floatToRawIntBits;
import static java.lang.Math.toIntExact;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector;
@ -155,10 +158,16 @@ public class TestSerDeUtils
Block actualString = toBinaryBlock(createUnboundedVarcharType(), "abdd", getInspector(String.class));
assertBlockEquals(actualString, expectedString);
// date
int date = toIntExact(LocalDate.of(2008, 10, 28).toEpochDay());
Block expectedDate = VARBINARY.createBlockBuilder(null, 1).writeInt(date).closeEntry().build();
Block actualDate = toBinaryBlock(BIGINT, Date.ofEpochDay(date), getInspector(Date.class));
assertBlockEquals(actualDate, expectedDate);
// timestamp
DateTime dateTime = new DateTime(2008, 10, 28, 16, 7, 15, 0);
Block expectedTimestamp = VARBINARY.createBlockBuilder(null, 1).writeLong(dateTime.getMillis()).closeEntry().build();
Block actualTimestamp = toBinaryBlock(BIGINT, new Timestamp(dateTime.getMillis()), getInspector(Timestamp.class));
Block actualTimestamp = toBinaryBlock(BIGINT, Timestamp.ofEpochMilli(dateTime.getMillis()), getInspector(Timestamp.class));
assertBlockEquals(actualTimestamp, expectedTimestamp);
// binary

188
presto-kafka/src/test/resources/read_test/all_datatypes_json.json Normal file
View File

@ -0,0 +1,188 @@
{
"tableName": "all_datatypes_json",
"schemaName": "product_tests",
"topicName": "all_datatypes_json",
"message": {
"dataFormat": "json",
"fields": [
{
"name": "c_varchar",
"type": "VARCHAR",
"mapping": "j_varchar"
},
{
"name": "c_bigint",
"type": "BIGINT",
"mapping": "j_bigint"
},
{
"name": "c_integer",
"type": "INTEGER",
"mapping": "j_integer"
},
{
"name": "c_smallint",
"type": "SMALLINT",
"mapping": "j_smallint"
},
{
"name": "c_tinyint",
"type": "TINYINT",
"mapping": "j_tinyint"
},
{
"name": "c_double",
"type": "DOUBLE",
"mapping": "j_double"
},
{
"name": "c_boolean",
"type": "BOOLEAN",
"mapping": "j_boolean"
},
{
"name": "c_timestamp_milliseconds_since_epoch",
"type": "TIMESTAMP",
"mapping": "j_timestamp_milliseconds_since_epoch",
"dataFormat": "milliseconds-since-epoch"
},
{
"name": "c_timestamp_seconds_since_epoch",
"type": "TIMESTAMP",
"mapping": "j_timestamp_seconds_since_epoch",
"dataFormat": "seconds-since-epoch"
},
{
"name": "c_timestamp_iso8601",
"type": "TIMESTAMP",
"mapping": "j_timestamp_iso8601",
"dataFormat": "iso8601"
},
{
"name": "c_timestamp_rfc2822",
"type": "TIMESTAMP",
"mapping": "j_timestamp_rfc2822",
"dataFormat": "rfc2822"
},
{
"name": "c_timestamp_custom",
"type": "TIMESTAMP",
"mapping": "j_timestamp_custom",
"dataFormat": "custom-date-time",
"formatHint": "MM/yyyy/dd H:m:s"
},
{
"name": "c_date_iso8601",
"type": "DATE",
"mapping": "j_date_iso8601",
"dataFormat": "iso8601"
},
{
"name": "c_date_rfc2822",
"type": "DATE",
"mapping": "j_date_rfc2822",
"dataFormat": "rfc2822"
},
{
"name": "c_date_custom",
"type": "DATE",
"mapping": "j_date_custom",
"dataFormat": "custom-date-time",
"formatHint": "yyyy/dd/MM"
},
{
"name": "c_time_milliseconds_since_epoch",
"type": "TIME",
"mapping": "j_time_milliseconds_since_epoch",
"dataFormat": "milliseconds-since-epoch"
},
{
"name": "c_time_seconds_since_epoch",
"type": "TIME",
"mapping": "j_time_seconds_since_epoch",
"dataFormat": "seconds-since-epoch"
},
{
"name": "c_time_iso8601",
"type": "TIME",
"mapping": "j_time_iso8601",
"dataFormat": "iso8601"
},
{
"name": "c_time_rfc2822",
"type": "TIME",
"mapping": "j_time_rfc2822",
"dataFormat": "rfc2822"
},
{
"name": "c_time_custom",
"type": "TIME",
"mapping": "j_time_custom",
"dataFormat": "custom-date-time",
"formatHint": "mm:HH:ss"
},
{
"name": "c_timestamptz_milliseconds_since_epoch",
"type": "TIMESTAMP WITH TIME ZONE",
"mapping": "j_timestamptz_milliseconds_since_epoch",
"dataFormat": "milliseconds-since-epoch"
},
{
"name": "c_timestamptz_seconds_since_epoch",
"type": "TIMESTAMP WITH TIME ZONE",
"mapping": "j_timestamptz_seconds_since_epoch",
"dataFormat": "seconds-since-epoch"
},
{
"name": "c_timestamptz_iso8601",
"type": "TIMESTAMP WITH TIME ZONE",
"mapping": "j_timestamptz_iso8601",
"dataFormat": "iso8601"
},
{
"name": "c_timestamptz_rfc2822",
"type": "TIMESTAMP WITH TIME ZONE",
"mapping": "j_timestamptz_rfc2822",
"dataFormat": "rfc2822"
},
{
"name": "c_timestamptz_custom",
"type": "TIMESTAMP WITH TIME ZONE",
"mapping": "j_timestamptz_custom",
"dataFormat": "custom-date-time",
"formatHint": "MM/yyyy/dd H:m:s"
},
{
"name": "c_timetz_milliseconds_since_epoch",
"type": "TIME WITH TIME ZONE",
"mapping": "j_timetz_milliseconds_since_epoch",
"dataFormat": "milliseconds-since-epoch"
},
{
"name": "c_timetz_seconds_since_epoch",
"type": "TIME WITH TIME ZONE",
"mapping": "j_timetz_seconds_since_epoch",
"dataFormat": "seconds-since-epoch"
},
{
"name": "c_timetz_iso8601",
"type": "TIME WITH TIME ZONE",
"mapping": "j_timetz_iso8601",
"dataFormat": "iso8601"
},
{
"name": "c_timetz_rfc2822",
"type": "TIME WITH TIME ZONE",
"mapping": "j_timetz_rfc2822",
"dataFormat": "rfc2822"
},
{
"name": "c_timetz_custom",
"type": "TIME WITH TIME ZONE",
"mapping": "j_timetz_custom",
"dataFormat": "custom-date-time",
"formatHint": "mm:HH:ss"
}
]
}
}

9
presto-main/src/main/java/io/prestosql/FullConnectorSession.java
View File

@ -39,7 +39,6 @@ public class FullConnectorSession
private final CatalogName catalogName;
private final String catalog;
private final SessionPropertyManager sessionPropertyManager;
private final boolean isLegacyTimestamp;
public FullConnectorSession(Session session, ConnectorIdentity identity)
{
@ -49,7 +48,6 @@ public class FullConnectorSession
this.catalogName = null;
this.catalog = null;
this.sessionPropertyManager = null;
this.isLegacyTimestamp = SystemSessionProperties.isLegacyTimestamp(session);
}
public FullConnectorSession(
@ -66,7 +64,6 @@ public class FullConnectorSession
this.catalogName = requireNonNull(catalogName, "catalogName is null");
this.catalog = requireNonNull(catalog, "catalog is null");
this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null");
this.isLegacyTimestamp = SystemSessionProperties.isLegacyTimestamp(session);
}
public Session getSession()
@ -116,12 +113,6 @@ public class FullConnectorSession
return session.getTraceToken();
}
@Override
public boolean isLegacyTimestamp()
{
return isLegacyTimestamp;
}
@Override
public <T> T getProperty(String propertyName, Class<T> type)
{

11
presto-main/src/main/java/io/prestosql/SystemSessionProperties.java
View File

@ -105,7 +105,6 @@ public final class SystemSessionProperties
public static final String ITERATIVE_OPTIMIZER_TIMEOUT = "iterative_optimizer_timeout";
public static final String ENABLE_FORCED_EXCHANGE_BELOW_GROUP_ID = "enable_forced_exchange_below_group_id";
public static final String EXCHANGE_COMPRESSION = "exchange_compression";
public static final String LEGACY_TIMESTAMP = "legacy_timestamp";
public static final String ENABLE_INTERMEDIATE_AGGREGATIONS = "enable_intermediate_aggregations";
public static final String PUSH_AGGREGATION_THROUGH_JOIN = "push_aggregation_through_join";
public static final String PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN = "push_partial_aggregation_through_join";
@ -519,11 +518,6 @@ public final class SystemSessionProperties
"Enable compression in exchanges",
featuresConfig.isExchangeCompressionEnabled(),
false),
booleanProperty(
LEGACY_TIMESTAMP,
"Use legacy TIME & TIMESTAMP semantics (warning: this will be removed)",
featuresConfig.isLegacyTimestamp(),
true),
booleanProperty(
ENABLE_INTERMEDIATE_AGGREGATIONS,
"Enable the use of intermediate aggregations",
@ -1066,11 +1060,6 @@ public final class SystemSessionProperties
return session.getSystemProperty(ITERATIVE_OPTIMIZER, Boolean.class);
}
public static boolean isLegacyTimestamp(Session session)
{
return session.getSystemProperty(LEGACY_TIMESTAMP, Boolean.class);
}
public static Duration getOptimizerTimeout(Session session)
{
return session.getSystemProperty(ITERATIVE_OPTIMIZER_TIMEOUT, Duration.class);

7
presto-main/src/main/java/io/prestosql/operator/scalar/ArrayToJsonCast.java
View File

@ -22,7 +22,6 @@ import io.prestosql.metadata.BoundVariables;
import io.prestosql.metadata.FunctionAndTypeManager;
import io.prestosql.metadata.SqlOperator;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.function.BuiltInScalarFunctionImplementation;
import io.prestosql.spi.function.OperatorType;
import io.prestosql.spi.type.StandardTypes;
@ -50,7 +49,7 @@ public class ArrayToJsonCast
extends SqlOperator
{
public static final ArrayToJsonCast ARRAY_TO_JSON = new ArrayToJsonCast();
private static final MethodHandle METHOD_HANDLE = methodHandle(ArrayToJsonCast.class, "toJson", JsonGeneratorWriter.class, ConnectorSession.class, Block.class);
private static final MethodHandle METHOD_HANDLE = methodHandle(ArrayToJsonCast.class, "toJson", JsonGeneratorWriter.class, Block.class);
private ArrayToJsonCast()
{
@ -79,14 +78,14 @@ public class ArrayToJsonCast
methodHandle);
}
public static Slice toJson(JsonGeneratorWriter writer, ConnectorSession session, Block block)
public static Slice toJson(JsonGeneratorWriter writer, Block block)
{
try {
SliceOutput output = new DynamicSliceOutput(40);
try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) {
jsonGenerator.writeStartArray();
for (int i = 0; i < block.getPositionCount(); i++) {
writer.writeJsonValue(jsonGenerator, block, i, session);
writer.writeJsonValue(jsonGenerator, block, i);
}
jsonGenerator.writeEndArray();
}

182
presto-main/src/main/java/io/prestosql/operator/scalar/DateTimeFunctions.java
View File

@ -107,14 +107,6 @@ public final class DateTimeFunctions
// We do all calculation in UTC, as session.getStartTime() is in UTC
// and we need to have UTC millis for packDateTimeWithZone
long millis = UTC_CHRONOLOGY.millisOfDay().get(session.getStartTime());
if (!session.isLegacyTimestamp()) {
// However, those UTC millis are pointing to the correct UTC timestamp
// Our TIME WITH TIME ZONE representation does use UTC 1970-01-01 representation
// So we have to hack here in order to get valid representation
// of TIME WITH TIME ZONE
millis -= valueToSessionTimeZoneOffsetDiff(session.getStartTime(), getDateTimeZone(session.getTimeZoneKey()));
}
return packDateTimeWithZone(millis, session.getTimeZoneKey());
}
@ -123,9 +115,6 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.TIME)
public static long localTime(ConnectorSession session)
{
if (session.isLegacyTimestamp()) {
return UTC_CHRONOLOGY.millisOfDay().get(session.getStartTime());
}
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
return localChronology.millisOfDay().get(session.getStartTime());
}
@ -151,9 +140,6 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.TIMESTAMP)
public static long localTimestamp(ConnectorSession session)
{
if (session.isLegacyTimestamp()) {
return session.getStartTime();
}
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
return localChronology.getZone().convertUTCToLocal(session.getStartTime());
}
@ -208,16 +194,9 @@ public final class DateTimeFunctions
// the maximum year represented by 64bits timestamp is ~584944387 it may require up to 35 characters.
public static Slice toISO8601FromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
DateTimeFormatter formatter = ISODateTimeFormat.dateTime()
.withChronology(getChronology(session.getTimeZoneKey()));
return utf8Slice(formatter.print(timestamp));
}
else {
DateTimeFormatter formatter = ISODateTimeFormat.dateHourMinuteSecondMillis()
.withChronology(UTC_CHRONOLOGY);
return utf8Slice(formatter.print(timestamp));
}
DateTimeFormatter formatter = ISODateTimeFormat.dateHourMinuteSecondMillis()
.withChronology(UTC_CHRONOLOGY);
return utf8Slice(formatter.print(timestamp));
}
@ScalarFunction("to_iso8601")
@ -317,12 +296,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.TIME)
public static long truncateTime(ConnectorSession session, @SqlType("varchar(x)") Slice unit, @SqlType(StandardTypes.TIME) long time)
{
if (session.isLegacyTimestamp()) {
return getTimeField(getChronology(session.getTimeZoneKey()), unit).roundFloor(time);
}
else {
return getTimeField(UTC_CHRONOLOGY, unit).roundFloor(time);
}
return getTimeField(UTC_CHRONOLOGY, unit).roundFloor(time);
}
@Description("truncate to the specified precision")
@ -341,12 +315,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.TIMESTAMP)
public static long truncateTimestamp(ConnectorSession session, @SqlType("varchar(x)") Slice unit, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getTimestampField(getChronology(session.getTimeZoneKey()), unit).roundFloor(timestamp);
}
else {
return getTimestampField(UTC_CHRONOLOGY, unit).roundFloor(timestamp);
}
return getTimestampField(UTC_CHRONOLOGY, unit).roundFloor(timestamp);
}
@Description("truncate to the specified precision")
@ -375,11 +344,6 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.TIME)
public static long addFieldValueTime(ConnectorSession session, @SqlType("varchar(x)") Slice unit, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.TIME) long time)
{
if (session.isLegacyTimestamp()) {
ISOChronology chronology = getChronology(session.getTimeZoneKey());
return modulo24Hour(chronology, getTimeField(chronology, unit).add(time, toIntExact(value)));
}
return modulo24Hour(getTimeField(UTC_CHRONOLOGY, unit).add(time, toIntExact(value)));
}
@ -407,10 +371,6 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.BIGINT) long value,
@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getTimestampField(getChronology(session.getTimeZoneKey()), unit).add(timestamp, toIntExact(value));
}
return getTimestampField(UTC_CHRONOLOGY, unit).add(timestamp, toIntExact(value));
}
@ -442,12 +402,6 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.BIGINT)
public static long diffTime(ConnectorSession session, @SqlType("varchar(x)") Slice unit, @SqlType(StandardTypes.TIME) long time1, @SqlType(StandardTypes.TIME) long time2)
{
if (session.isLegacyTimestamp()) {
// Session zone could have policy change on/around 1970-01-01, so we cannot use UTC
ISOChronology chronology = getChronology(session.getTimeZoneKey());
return getTimeField(chronology, unit).getDifferenceAsLong(time2, time1);
}
return getTimeField(UTC_CHRONOLOGY, unit).getDifferenceAsLong(time2, time1);
}
@ -473,10 +427,6 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.TIMESTAMP) long timestamp1,
@SqlType(StandardTypes.TIMESTAMP) long timestamp2)
{
if (session.isLegacyTimestamp()) {
return getTimestampField(getChronology(session.getTimeZoneKey()), unit).getDifferenceAsLong(timestamp2, timestamp1);
}
return getTimestampField(UTC_CHRONOLOGY, unit).getDifferenceAsLong(timestamp2, timestamp1);
}
@ -587,16 +537,11 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.VARCHAR)
public static Slice formatDatetime(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp, @SqlType("varchar(x)") Slice formatString)
{
if (session.isLegacyTimestamp()) {
return formatDatetime(getChronology(session.getTimeZoneKey()), session.getLocale(), timestamp, formatString);
}
else {
if (datetimeFormatSpecifiesZone(formatString)) {
// Timezone is unknown for TIMESTAMP w/o TZ so it cannot be printed out.
throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "format_datetime for TIMESTAMP type, cannot use 'Z' nor 'z' in format, as this type does not contain TZ information");
}
return formatDatetime(UTC_CHRONOLOGY, session.getLocale(), timestamp, formatString);
if (datetimeFormatSpecifiesZone(formatString)) {
// Timezone is unknown for TIMESTAMP w/o TZ so it cannot be printed out.
throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "format_datetime for TIMESTAMP type, cannot use 'Z' nor 'z' in format, as this type does not contain TZ information");
}
return formatDatetime(UTC_CHRONOLOGY, session.getLocale(), timestamp, formatString);
}
/**
@ -657,12 +602,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.VARCHAR)
public static Slice dateFormat(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp, @SqlType("varchar(x)") Slice formatString)
{
if (session.isLegacyTimestamp()) {
return dateFormat(getChronology(session.getTimeZoneKey()), session.getLocale(), timestamp, formatString);
}
else {
return dateFormat(UTC_CHRONOLOGY, session.getLocale(), timestamp, formatString);
}
return dateFormat(UTC_CHRONOLOGY, session.getLocale(), timestamp, formatString);
}
@ScalarFunction("date_format")
@ -691,7 +631,7 @@ public final class DateTimeFunctions
public static long dateParse(ConnectorSession session, @SqlType("varchar(x)") Slice dateTime, @SqlType("varchar(y)") Slice formatString)
{
DateTimeFormatter formatter = DATETIME_FORMATTER_CACHE.get(formatString)
.withChronology(session.isLegacyTimestamp() ? getChronology(session.getTimeZoneKey()) : UTC_CHRONOLOGY)
.withZoneUTC()
.withLocale(session.getLocale());
try {
@ -803,12 +743,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.BIGINT)
public static long minuteFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).minuteOfHour().get(timestamp);
}
else {
return MINUTE_OF_HOUR.get(timestamp);
}
return MINUTE_OF_HOUR.get(timestamp);
}
@Description("minute of the hour of the given timestamp")
@ -824,12 +759,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.BIGINT)
public static long minuteFromTime(ConnectorSession session, @SqlType(StandardTypes.TIME) long time)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).minuteOfHour().get(time);
}
else {
return MINUTE_OF_HOUR.get(time);
}
return MINUTE_OF_HOUR.get(time);
}
@Description("minute of the hour of the given time")
@ -853,12 +783,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.BIGINT)
public static long hourFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).hourOfDay().get(timestamp);
}
else {
return HOUR_OF_DAY.get(timestamp);
}
return HOUR_OF_DAY.get(timestamp);
}
@Description("hour of the day of the given timestamp")
@ -874,12 +799,7 @@ public final class DateTimeFunctions
@SqlType(StandardTypes.BIGINT)
public static long hourFromTime(ConnectorSession session, @SqlType(StandardTypes.TIME) long time)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).hourOfDay().get(time);
}
else {
return HOUR_OF_DAY.get(time);
}
return HOUR_OF_DAY.get(time);
}
@Description("hour of the day of the given time")
@ -901,14 +821,9 @@ public final class DateTimeFunctions
@Description("day of the week of the given timestamp")
@ScalarFunction(value = "day_of_week", alias = "dow")
@SqlType(StandardTypes.BIGINT)
public static long dayOfWeekFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long dayOfWeekFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).dayOfWeek().get(timestamp);
}
else {
return DAY_OF_WEEK.get(timestamp);
}
return DAY_OF_WEEK.get(timestamp);
}
@Description("day of the week of the given timestamp")
@ -930,14 +845,9 @@ public final class DateTimeFunctions
@Description("day of the month of the given timestamp")
@ScalarFunction(value = "day", alias = "day_of_month")
@SqlType(StandardTypes.BIGINT)
public static long dayFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long dayFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).dayOfMonth().get(timestamp);
}
else {
return DAY_OF_MONTH.get(timestamp);
}
return DAY_OF_MONTH.get(timestamp);
}
@Description("day of the month of the given timestamp")
@ -967,14 +877,9 @@ public final class DateTimeFunctions
@Description("day of the year of the given timestamp")
@ScalarFunction(value = "day_of_year", alias = "doy")
@SqlType(StandardTypes.BIGINT)
public static long dayOfYearFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long dayOfYearFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).dayOfYear().get(timestamp);
}
else {
return DAY_OF_YEAR.get(timestamp);
}
return DAY_OF_YEAR.get(timestamp);
}
@Description("day of the year of the given timestamp")
@ -996,14 +901,9 @@ public final class DateTimeFunctions
@Description("week of the year of the given timestamp")
@ScalarFunction(value = "week", alias = "week_of_year")
@SqlType(StandardTypes.BIGINT)
public static long weekFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long weekFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).weekOfWeekyear().get(timestamp);
}
else {
return WEEK_OF_YEAR.get(timestamp);
}
return WEEK_OF_YEAR.get(timestamp);
}
@Description("week of the year of the given timestamp")
@ -1025,14 +925,9 @@ public final class DateTimeFunctions
@Description("year of the ISO week of the given timestamp")
@ScalarFunction(value = "year_of_week", alias = "yow")
@SqlType(StandardTypes.BIGINT)
public static long yearOfWeekFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long yearOfWeekFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).weekyear().get(timestamp);
}
else {
return YEAR_OF_WEEK.get(timestamp);
}
return YEAR_OF_WEEK.get(timestamp);
}
@Description("year of the ISO week of the given timestamp")
@ -1054,14 +949,9 @@ public final class DateTimeFunctions
@Description("month of the year of the given timestamp")
@ScalarFunction("month")
@SqlType(StandardTypes.BIGINT)
public static long monthFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long monthFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).monthOfYear().get(timestamp);
}
else {
return MONTH_OF_YEAR.get(timestamp);
}
return MONTH_OF_YEAR.get(timestamp);
}
@Description("month of the year of the given timestamp")
@ -1091,14 +981,9 @@ public final class DateTimeFunctions
@Description("quarter of the year of the given timestamp")
@ScalarFunction("quarter")
@SqlType(StandardTypes.BIGINT)
public static long quarterFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long quarterFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return QUARTER_OF_YEAR.getField(getChronology(session.getTimeZoneKey())).get(timestamp);
}
else {
return QUARTER_OF_YEAR.getField(UTC_CHRONOLOGY).get(timestamp);
}
return QUARTER_OF_YEAR.getField(UTC_CHRONOLOGY).get(timestamp);
}
@Description("quarter of the year of the given timestamp")
@ -1120,14 +1005,9 @@ public final class DateTimeFunctions
@Description("year of the given timestamp")
@ScalarFunction("year")
@SqlType(StandardTypes.BIGINT)
public static long yearFromTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long timestamp)
public static long yearFromTimestamp(@SqlType(StandardTypes.TIMESTAMP) long timestamp)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).year().get(timestamp);
}
else {
return YEAR.get(timestamp);
}
return YEAR.get(timestamp);
}
@Description("year of the given timestamp")

13
presto-main/src/main/java/io/prestosql/operator/scalar/FormatFunction.java
View File

@ -197,7 +197,7 @@ public final class FormatFunction
return (session, block) -> toZonedDateTime(type.getLong(block, position));
}
if (type.equals(TIMESTAMP)) {
return (session, block) -> toLocalDateTime(session, type.getLong(block, position));
return (session, block) -> toLocalDateTime(type.getLong(block, position));
}
if (type.equals(TIME)) {
return (session, block) -> toLocalTime(session, type.getLong(block, position));
@ -266,23 +266,14 @@ public final class FormatFunction
return ZonedDateTime.ofInstant(instant, zoneId);
}
private static LocalDateTime toLocalDateTime(ConnectorSession session, long value)
private static LocalDateTime toLocalDateTime(long value)
{
Instant instant = Instant.ofEpochMilli(value);
if (session.isLegacyTimestamp()) {
ZoneId zoneId = ZoneId.of(session.getTimeZoneKey().getId());
return LocalDateTime.ofInstant(instant, zoneId);
}
return LocalDateTime.ofInstant(instant, ZoneOffset.UTC);
}
private static LocalTime toLocalTime(ConnectorSession session, long value)
{
if (session.isLegacyTimestamp()) {
Instant instant = Instant.ofEpochMilli(value);
ZoneId zoneId = ZoneId.of(session.getTimeZoneKey().getId());
return ZonedDateTime.ofInstant(instant, zoneId).toLocalTime();
}
return LocalTime.ofNanoOfDay(MILLISECONDS.toNanos(value));
}

2
presto-main/src/main/java/io/prestosql/operator/scalar/JsonOperators.java
View File

@ -337,7 +337,7 @@ public final class JsonOperators
try {
SliceOutput output = new DynamicSliceOutput(25);
try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) {
jsonGenerator.writeString(printTimestampWithoutTimeZone(session.getTimeZoneKey(), value));
jsonGenerator.writeString(printTimestampWithoutTimeZone(value));
}
return output.slice();
}

7
presto-main/src/main/java/io/prestosql/operator/scalar/MapToJsonCast.java
View File

@ -23,7 +23,6 @@ import io.prestosql.metadata.FunctionAndTypeManager;
import io.prestosql.metadata.SqlOperator;
import io.prestosql.spi.annotation.UsedByGeneratedCode;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.function.BuiltInScalarFunctionImplementation;
import io.prestosql.spi.function.OperatorType;
import io.prestosql.spi.type.StandardTypes;
@ -54,7 +53,7 @@ public class MapToJsonCast
extends SqlOperator
{
public static final MapToJsonCast MAP_TO_JSON = new MapToJsonCast();
private static final MethodHandle METHOD_HANDLE = methodHandle(MapToJsonCast.class, "toJson", ObjectKeyProvider.class, JsonGeneratorWriter.class, ConnectorSession.class, Block.class);
private static final MethodHandle METHOD_HANDLE = methodHandle(MapToJsonCast.class, "toJson", ObjectKeyProvider.class, JsonGeneratorWriter.class, Block.class);
private MapToJsonCast()
{
@ -87,7 +86,7 @@ public class MapToJsonCast
}
@UsedByGeneratedCode
public static Slice toJson(ObjectKeyProvider provider, JsonGeneratorWriter writer, ConnectorSession session, Block block)
public static Slice toJson(ObjectKeyProvider provider, JsonGeneratorWriter writer, Block block)
{
try {
Map<String, Integer> orderedKeyToValuePosition = new TreeMap<>();
@ -101,7 +100,7 @@ public class MapToJsonCast
jsonGenerator.writeStartObject();
for (Map.Entry<String, Integer> entry : orderedKeyToValuePosition.entrySet()) {
jsonGenerator.writeFieldName(entry.getKey());
writer.writeJsonValue(jsonGenerator, block, entry.getValue(), session);
writer.writeJsonValue(jsonGenerator, block, entry.getValue());
}
jsonGenerator.writeEndObject();
}

7
presto-main/src/main/java/io/prestosql/operator/scalar/RowToJsonCast.java
View File

@ -23,7 +23,6 @@ import io.prestosql.metadata.FunctionAndTypeManager;
import io.prestosql.metadata.SqlOperator;
import io.prestosql.spi.annotation.UsedByGeneratedCode;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.function.BuiltInScalarFunctionImplementation;
import io.prestosql.spi.function.OperatorType;
import io.prestosql.spi.type.StandardTypes;
@ -53,7 +52,7 @@ public class RowToJsonCast
extends SqlOperator
{
public static final RowToJsonCast ROW_TO_JSON = new RowToJsonCast();
private static final MethodHandle METHOD_HANDLE = methodHandle(RowToJsonCast.class, "toJson", List.class, ConnectorSession.class, Block.class);
private static final MethodHandle METHOD_HANDLE = methodHandle(RowToJsonCast.class, "toJson", List.class, Block.class);
private RowToJsonCast()
{
@ -85,14 +84,14 @@ public class RowToJsonCast
}
@UsedByGeneratedCode
public static Slice toJson(List<JsonGeneratorWriter> fieldWriters, ConnectorSession session, Block block)
public static Slice toJson(List<JsonGeneratorWriter> fieldWriters, Block block)
{
try {
SliceOutput output = new DynamicSliceOutput(40);
try (JsonGenerator jsonGenerator = createJsonGenerator(JSON_FACTORY, output)) {
jsonGenerator.writeStartArray();
for (int i = 0; i < block.getPositionCount(); i++) {
fieldWriters.get(i).writeJsonValue(jsonGenerator, block, i, session);
fieldWriters.get(i).writeJsonValue(jsonGenerator, block, i);
}
jsonGenerator.writeEndArray();
}

2
presto-main/src/main/java/io/prestosql/operator/scalar/SequenceFunction.java
View File

@ -137,7 +137,7 @@ public final class SequenceFunction
int value = 0;
for (int i = 0; i < length; ++i) {
BIGINT.writeLong(blockBuilder, DateTimeOperators.timestampPlusIntervalYearToMonth(session, start, value));
BIGINT.writeLong(blockBuilder, DateTimeOperators.timestampPlusIntervalYearToMonth(start, value));
value += step;
}

7
presto-main/src/main/java/io/prestosql/sql/analyzer/ExpressionAnalyzer.java
View File

@ -881,12 +881,7 @@ public class ExpressionAnalyzer
protected Type visitTimestampLiteral(TimestampLiteral node, StackableAstVisitorContext<Context> context)
{
try {
if (SystemSessionProperties.isLegacyTimestamp(session)) {
parseTimestampLiteral(session.getTimeZoneKey(), node.getValue());
}
else {
parseTimestampLiteral(node.getValue());
}
parseTimestampLiteral(node.getValue());
}
catch (Exception e) {
throw new SemanticException(INVALID_LITERAL, node, "'%s' is not a valid timestamp literal", node.getValue());

17
presto-main/src/main/java/io/prestosql/sql/analyzer/FeaturesConfig.java
View File

@ -57,7 +57,9 @@ import static java.util.concurrent.TimeUnit.MINUTES;
"analyzer.experimental-syntax-enabled",
"optimizer.processing-optimization",
"deprecated.legacy-order-by",
"deprecated.legacy-join-using"})
"deprecated.legacy-join-using",
"deprecated.legacy-timestamp",
})
public class FeaturesConfig
{
@VisibleForTesting
@ -92,7 +94,6 @@ public class FeaturesConfig
private boolean pushLimitThroughSemiJoin = true;
private boolean pushLimitThroughOuterJoin = true;
private boolean exchangeCompressionEnabled;
private boolean legacyTimestamp = true;
private boolean legacyMapSubscript;
private boolean optimizeMixedDistinctAggregations;
private boolean unwrapCasts = true;
@ -255,18 +256,6 @@ public class FeaturesConfig
return this;
}
@Config("deprecated.legacy-timestamp")
public FeaturesConfig setLegacyTimestamp(boolean value)
{
this.legacyTimestamp = value;
return this;
}
public boolean isLegacyTimestamp()
{
return legacyTimestamp;
}
@Config("deprecated.legacy-map-subscript")
public FeaturesConfig setLegacyMapSubscript(boolean value)
{

14
presto-main/src/main/java/io/prestosql/sql/planner/LiteralInterpreter.java
View File

@ -256,24 +256,14 @@ public final class LiteralInterpreter
@Override
protected Long visitTimeLiteral(TimeLiteral node, ConnectorSession session)
{
if (session.isLegacyTimestamp()) {
return parseTimeLiteral(session.getTimeZoneKey(), node.getValue());
}
else {
return parseTimeLiteral(node.getValue());
}
return parseTimeLiteral(node.getValue());
}
@Override
protected Long visitTimestampLiteral(TimestampLiteral node, ConnectorSession session)
{
try {
if (session.isLegacyTimestamp()) {
return parseTimestampLiteral(session.getTimeZoneKey(), node.getValue());
}
else {
return parseTimestampLiteral(node.getValue());
}
return parseTimestampLiteral(node.getValue());
}
catch (RuntimeException e) {
throw new SemanticException(INVALID_LITERAL, node, "'%s' is not a valid timestamp literal", node.getValue());

26
presto-main/src/main/java/io/prestosql/sql/relational/SqlToRowExpressionTranslator.java
View File

@ -17,7 +17,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import io.prestosql.Session;
import io.prestosql.SystemSessionProperties;
import io.prestosql.metadata.FunctionAndTypeManager;
import io.prestosql.operator.scalar.TryFunction;
import io.prestosql.spi.function.FunctionKind;
@ -33,7 +32,6 @@ import io.prestosql.spi.type.DecimalParseResult;
import io.prestosql.spi.type.Decimals;
import io.prestosql.spi.type.RowType;
import io.prestosql.spi.type.RowType.Field;
import io.prestosql.spi.type.TimeZoneKey;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeSignature;
import io.prestosql.spi.type.UnknownType;
@ -174,8 +172,6 @@ public final class SqlToRowExpressionTranslator
types,
layout,
functionAndTypeManager,
session.getTimeZoneKey(),
SystemSessionProperties.isLegacyTimestamp(session),
transactionId);
RowExpression result = visitor.process(expression, null);
@ -196,8 +192,6 @@ public final class SqlToRowExpressionTranslator
private final Map<NodeRef<Expression>, Type> types;
private final Map<Symbol, Integer> layout;
private final FunctionAndTypeManager functionAndTypeManager;
private final TimeZoneKey timeZoneKey;
private final boolean isLegacyTimestamp;
private final Optional<TransactionId> transactionId;
private final FunctionResolution functionResolution;
@ -206,16 +200,12 @@ public final class SqlToRowExpressionTranslator
Map<NodeRef<Expression>, Type> types,
Map<Symbol, Integer> layout,
FunctionAndTypeManager functionAndTypeManager,
TimeZoneKey timeZoneKey,
boolean isLegacyTimestamp,
Optional<TransactionId> transactionId)
{
this.functionKind = functionKind;
this.types = ImmutableMap.copyOf(requireNonNull(types, "types is null"));
this.functionAndTypeManager = functionAndTypeManager;
this.layout = layout;
this.timeZoneKey = timeZoneKey;
this.isLegacyTimestamp = isLegacyTimestamp;
this.transactionId = transactionId;
this.functionResolution = new FunctionResolution(functionAndTypeManager);
}
@ -345,13 +335,7 @@ public final class SqlToRowExpressionTranslator
value = parseTimeWithTimeZone(node.getValue());
}
else {
if (isLegacyTimestamp) {
// parse in time zone of client
value = parseTimeWithoutTimeZone(timeZoneKey, node.getValue());
}
else {
value = parseTimeWithoutTimeZone(node.getValue());
}
value = parseTimeWithoutTimeZone(node.getValue());
}
return constant(value, getType(node));
}
@ -359,13 +343,7 @@ public final class SqlToRowExpressionTranslator
@Override
protected RowExpression visitTimestampLiteral(TimestampLiteral node, Void context)
{
long value;
if (isLegacyTimestamp) {
value = parseTimestampLiteral(timeZoneKey, node.getValue());
}
else {
value = parseTimestampLiteral(node.getValue());
}
long value = parseTimestampLiteral(node.getValue());
return constant(value, getType(node));
}

73
presto-main/src/main/java/io/prestosql/testing/DateTimeTestingUtils.java
View File

@ -13,22 +13,14 @@
*/
package io.prestosql.testing;
import io.prestosql.Session;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.type.SqlTime;
import io.prestosql.spi.type.SqlTimestamp;
import io.prestosql.spi.type.TimeZoneKey;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import static io.prestosql.spi.util.DateTimeZoneIndex.getDateTimeZone;
import static java.lang.Math.toIntExact;
import static java.time.ZoneOffset.UTC;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@ -44,37 +36,8 @@ public final class DateTimeTestingUtils
int hourOfDay,
int minuteOfHour,
int secondOfMinute,
int millisOfSecond,
Session session)
int millisOfSecond)
{
return sqlTimestampOf(
year,
monthOfYear,
dayOfMonth,
hourOfDay,
minuteOfHour,
secondOfMinute,
millisOfSecond,
getDateTimeZone(session.getTimeZoneKey()),
session.getTimeZoneKey(),
session.toConnectorSession());
}
public static SqlTimestamp sqlTimestampOf(
int year,
int monthOfYear,
int dayOfMonth,
int hourOfDay,
int minuteOfHour,
int secondOfMinute,
int millisOfSecond,
DateTimeZone baseZone,
TimeZoneKey timestampZone,
ConnectorSession session)
{
if (session.isLegacyTimestamp()) {
return new SqlTimestamp(new DateTime(year, monthOfYear, dayOfMonth, hourOfDay, minuteOfHour, secondOfMinute, millisOfSecond, baseZone).getMillis(), timestampZone);
}
return sqlTimestampOf(LocalDateTime.of(year, monthOfYear, dayOfMonth, hourOfDay, minuteOfHour, secondOfMinute, millisToNanos(millisOfSecond)));
}
@ -86,48 +49,28 @@ public final class DateTimeTestingUtils
return new SqlTimestamp(DAYS.toMillis(dateTime.toLocalDate().toEpochDay()) + NANOSECONDS.toMillis(dateTime.toLocalTime().toNanoOfDay()));
}
public static SqlTimestamp sqlTimestampOf(DateTime dateTime, Session session)
private static SqlTimestamp sqlTimestampOf(DateTime dateTime)
{
return sqlTimestampOf(dateTime, session.toConnectorSession());
return sqlTimestampOf(dateTime.getMillis());
}
private static SqlTimestamp sqlTimestampOf(DateTime dateTime, ConnectorSession session)
public static SqlTimestamp sqlTimestampOf(long millis)
{
return sqlTimestampOf(dateTime.getMillis(), session);
}
public static SqlTimestamp sqlTimestampOf(long millis, ConnectorSession session)
{
if (session.isLegacyTimestamp()) {
return new SqlTimestamp(millis, session.getTimeZoneKey());
}
else {
return new SqlTimestamp(millis);
}
return new SqlTimestamp(millis);
}
public static SqlTime sqlTimeOf(
int hourOfDay,
int minuteOfHour,
int secondOfMinute,
int millisOfSecond,
Session session)
int millisOfSecond)
{
LocalTime time = LocalTime.of(hourOfDay, minuteOfHour, secondOfMinute, millisToNanos(millisOfSecond));
return sqlTimeOf(time, session);
return sqlTimeOf(time);
}
public static SqlTime sqlTimeOf(LocalTime time, Session session)
public static SqlTime sqlTimeOf(LocalTime time)
{
if (session.toConnectorSession().isLegacyTimestamp()) {
long millisUtc = LocalDate.ofEpochDay(0)
.atTime(time)
.atZone(UTC)
.withZoneSameLocal(ZoneId.of(session.getTimeZoneKey().getId()))
.toInstant()
.toEpochMilli();
return new SqlTime(millisUtc, session.getTimeZoneKey());
}
return new SqlTime(NANOSECONDS.toMillis(time.toNanoOfDay()));
}

2
presto-main/src/main/java/io/prestosql/testing/MaterializedResult.java
View File

@ -302,7 +302,7 @@ public class MaterializedResult
type.writeLong(blockBuilder, packDateTimeWithZone(millisUtc, timeZoneKey));
}
else if (TIMESTAMP.equals(type)) {
long millisUtc = ((SqlTimestamp) value).getMillisUtc();
long millisUtc = ((SqlTimestamp) value).getMillis();
type.writeLong(blockBuilder, millisUtc);
}
else if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {

14
presto-main/src/main/java/io/prestosql/testing/TestingConnectorSession.java
View File

@ -22,7 +22,6 @@ import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.security.ConnectorIdentity;
import io.prestosql.spi.session.PropertyMetadata;
import io.prestosql.spi.type.TimeZoneKey;
import io.prestosql.sql.analyzer.FeaturesConfig;
import java.util.List;
import java.util.Locale;
@ -50,7 +49,6 @@ public class TestingConnectorSession
private final long startTime;
private final Map<String, PropertyMetadata<?>> properties;
private final Map<String, Object> propertyValues;
private final boolean isLegacyTimestamp;
public TestingConnectorSession(List<PropertyMetadata<?>> properties)
{
@ -59,7 +57,7 @@ public class TestingConnectorSession
public TestingConnectorSession(List<PropertyMetadata<?>> properties, Map<String, Object> propertyValues)
{
this("user", Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, propertyValues, new FeaturesConfig().isLegacyTimestamp());
this("user", Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, propertyValues);
}
public TestingConnectorSession(
@ -70,8 +68,7 @@ public class TestingConnectorSession
Locale locale,
long startTime,
List<PropertyMetadata<?>> propertyMetadatas,
Map<String, Object> propertyValues,
boolean isLegacyTimestamp)
Map<String, Object> propertyValues)
{
this.queryId = queryIdGenerator.createNextQueryId().toString();
this.identity = new ConnectorIdentity(requireNonNull(user, "user is null"), Optional.empty(), Optional.empty());
@ -82,7 +79,6 @@ public class TestingConnectorSession
this.startTime = startTime;
this.properties = Maps.uniqueIndex(propertyMetadatas, PropertyMetadata::getName);
this.propertyValues = ImmutableMap.copyOf(propertyValues);
this.isLegacyTimestamp = isLegacyTimestamp;
}
@Override
@ -127,12 +123,6 @@ public class TestingConnectorSession
return traceToken;
}
@Override
public boolean isLegacyTimestamp()
{
return isLegacyTimestamp;
}
@Override
public <T> T getProperty(String name, Class<T> type)
{

12
presto-main/src/main/java/io/prestosql/type/DateOperators.java
View File

@ -114,17 +114,7 @@ public final class DateOperators
@SqlType(StandardTypes.TIMESTAMP)
public static long castToTimestamp(ConnectorSession session, @SqlType(StandardTypes.DATE) long value)
{
if (session.isLegacyTimestamp()) {
long utcMillis = TimeUnit.DAYS.toMillis(value);
// date is encoded as milliseconds at midnight in UTC
// convert to midnight in the session timezone
ISOChronology chronology = getChronology(session.getTimeZoneKey());
return utcMillis - chronology.getZone().getOffset(utcMillis);
}
else {
return TimeUnit.DAYS.toMillis(value);
}
return TimeUnit.DAYS.toMillis(value);
}
@ScalarOperator(CAST)

27
presto-main/src/main/java/io/prestosql/type/DateTimeOperators.java
View File

@ -162,26 +162,16 @@ public final class DateTimeOperators
@ScalarOperator(ADD)
@SqlType(StandardTypes.TIMESTAMP)
public static long timestampPlusIntervalYearToMonth(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long left, @SqlType(StandardTypes.INTERVAL_YEAR_TO_MONTH) long right)
public static long timestampPlusIntervalYearToMonth(@SqlType(StandardTypes.TIMESTAMP) long left, @SqlType(StandardTypes.INTERVAL_YEAR_TO_MONTH) long right)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).monthOfYear().add(left, right);
}
else {
return MONTH_OF_YEAR_UTC.add(left, right);
}
return MONTH_OF_YEAR_UTC.add(left, right);
}
@ScalarOperator(ADD)
@SqlType(StandardTypes.TIMESTAMP)
public static long intervalYearToMonthPlusTimestamp(ConnectorSession session, @SqlType(StandardTypes.INTERVAL_YEAR_TO_MONTH) long left, @SqlType(StandardTypes.TIMESTAMP) long right)
public static long intervalYearToMonthPlusTimestamp(@SqlType(StandardTypes.INTERVAL_YEAR_TO_MONTH) long left, @SqlType(StandardTypes.TIMESTAMP) long right)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).monthOfYear().add(right, left);
}
else {
return MONTH_OF_YEAR_UTC.add(right, left);
}
return MONTH_OF_YEAR_UTC.add(right, left);
}
@ScalarOperator(ADD)
@ -260,14 +250,9 @@ public final class DateTimeOperators
@ScalarOperator(SUBTRACT)
@SqlType(StandardTypes.TIMESTAMP)
public static long timestampMinusIntervalYearToMonth(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long left, @SqlType(StandardTypes.INTERVAL_YEAR_TO_MONTH) long right)
public static long timestampMinusIntervalYearToMonth(@SqlType(StandardTypes.TIMESTAMP) long left, @SqlType(StandardTypes.INTERVAL_YEAR_TO_MONTH) long right)
{
if (session.isLegacyTimestamp()) {
return getChronology(session.getTimeZoneKey()).monthOfYear().add(left, -right);
}
else {
return MONTH_OF_YEAR_UTC.add(left, -right);
}
return MONTH_OF_YEAR_UTC.add(left, -right);
}
@ScalarOperator(SUBTRACT)

33
presto-main/src/main/java/io/prestosql/type/TimeOperators.java
View File

@ -118,19 +118,14 @@ public final class TimeOperators
@SqlType(StandardTypes.TIME_WITH_TIME_ZONE)
public static long castToTimeWithTimeZone(ConnectorSession session, @SqlType(StandardTypes.TIME) long value)
{
if (session.isLegacyTimestamp()) {
return packDateTimeWithZone(value, session.getTimeZoneKey());
}
else {
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
// This cast does treat TIME as wall time in session TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
// We use value offset in this place to be sure that we will have same hour represented
// in TIME WITH TIME ZONE. Calculating real TZ offset will happen when really required.
// This is done due to inadequate TIME WITH TIME ZONE representation.
return packDateTimeWithZone(localChronology.getZone().convertLocalToUTC(value, false), session.getTimeZoneKey());
}
// This cast does treat TIME as wall time in session TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
// We use value offset in this place to be sure that we will have same hour represented
// in TIME WITH TIME ZONE. Calculating real TZ offset will happen when really required.
// This is done due to inadequate TIME WITH TIME ZONE representation.
return packDateTimeWithZone(localChronology.getZone().convertLocalToUTC(value, false), session.getTimeZoneKey());
}
@ScalarOperator(CAST)
@ -152,12 +147,7 @@ public final class TimeOperators
@SqlType("varchar(x)")
public static Slice castToSlice(ConnectorSession session, @SqlType(StandardTypes.TIME) long value)
{
if (session.isLegacyTimestamp()) {
return utf8Slice(printTimeWithoutTimeZone(session.getTimeZoneKey(), value));
}
else {
return utf8Slice(printTimeWithoutTimeZone(value));
}
return utf8Slice(printTimeWithoutTimeZone(value));
}
@ScalarOperator(CAST)
@ -166,12 +156,7 @@ public final class TimeOperators
public static long castFromSlice(ConnectorSession session, @SqlType("varchar(x)") Slice value)
{
try {
if (session.isLegacyTimestamp()) {
return parseTimeWithoutTimeZone(session.getTimeZoneKey(), value.toStringUtf8());
}
else {
return parseTimeWithoutTimeZone(value.toStringUtf8());
}
return parseTimeWithoutTimeZone(value.toStringUtf8());
}
catch (IllegalArgumentException e) {
throw new PrestoException(INVALID_CAST_ARGUMENT, "Value cannot be cast to time: " + value.toStringUtf8(), e);

24
presto-main/src/main/java/io/prestosql/type/TimeWithTimeZoneOperators.java
View File

@ -16,7 +16,6 @@ package io.prestosql.type;
import io.airlift.slice.Slice;
import io.airlift.slice.XxHash64;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.function.BlockIndex;
import io.prestosql.spi.function.BlockPosition;
import io.prestosql.spi.function.IsNull;
@ -121,29 +120,24 @@ public final class TimeWithTimeZoneOperators
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIME)
public static long castToTime(ConnectorSession session, @SqlType(StandardTypes.TIME_WITH_TIME_ZONE) long value)
public static long castToTime(@SqlType(StandardTypes.TIME_WITH_TIME_ZONE) long value)
{
// This is exactly the same operation as for TIME WITH TIME ZONE -> TIMESTAMP, as the representations
// of those types are aligned in range that is covered by TIME WITH TIME ZONE.
return castToTimestamp(session, value);
return castToTimestamp(value);
}
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIMESTAMP)
public static long castToTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIME_WITH_TIME_ZONE) long value)
public static long castToTimestamp(@SqlType(StandardTypes.TIME_WITH_TIME_ZONE) long value)
{
if (session.isLegacyTimestamp()) {
return unpackMillisUtc(value);
}
else {
// This is hack that we need to use as the timezone interpretation depends on date (not only on time)
// TODO remove REFERENCE_TIMESTAMP_UTC when removing support for political time zones in TIME WIT TIME ZONE
long currentMillisOfDay = ChronoField.MILLI_OF_DAY.getFrom(Instant.ofEpochMilli(REFERENCE_TIMESTAMP_UTC).atZone(ZoneOffset.UTC));
long timeMillisUtcInCurrentDay = REFERENCE_TIMESTAMP_UTC - currentMillisOfDay + unpackMillisUtc(value);
// This is hack that we need to use as the timezone interpretation depends on date (not only on time)
// TODO remove REFERENCE_TIMESTAMP_UTC when removing support for political time zones in TIME WIT TIME ZONE
long currentMillisOfDay = ChronoField.MILLI_OF_DAY.getFrom(Instant.ofEpochMilli(REFERENCE_TIMESTAMP_UTC).atZone(ZoneOffset.UTC));
long timeMillisUtcInCurrentDay = REFERENCE_TIMESTAMP_UTC - currentMillisOfDay + unpackMillisUtc(value);
ISOChronology chronology = getChronology(unpackZoneKey(value));
return unpackMillisUtc(value) + chronology.getZone().getOffset(timeMillisUtcInCurrentDay);
}
ISOChronology chronology = getChronology(unpackZoneKey(value));
return unpackMillisUtc(value) + chronology.getZone().getOffset(timeMillisUtcInCurrentDay);
}
@ScalarOperator(CAST)

81
presto-main/src/main/java/io/prestosql/type/TimestampOperators.java
View File

@ -122,102 +122,59 @@ public final class TimestampOperators
@ScalarFunction("date")
@ScalarOperator(CAST)
@SqlType(StandardTypes.DATE)
public static long castToDate(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long value)
public static long castToDate(@SqlType(StandardTypes.TIMESTAMP) long value)
{
ISOChronology chronology;
if (session.isLegacyTimestamp()) {
// round down the current timestamp to days
chronology = getChronology(session.getTimeZoneKey());
long date = chronology.dayOfYear().roundFloor(value);
// date is currently midnight in timezone of the session
// convert to UTC
long millis = date + chronology.getZone().getOffset(date);
return TimeUnit.MILLISECONDS.toDays(millis);
}
else {
return TimeUnit.MILLISECONDS.toDays(value);
}
return TimeUnit.MILLISECONDS.toDays(value);
}
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIME)
public static long castToTime(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long value)
public static long castToTime(@SqlType(StandardTypes.TIMESTAMP) long value)
{
if (session.isLegacyTimestamp()) {
return modulo24Hour(getChronology(session.getTimeZoneKey()), value);
}
else {
return modulo24Hour(value);
}
return modulo24Hour(value);
}
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIME_WITH_TIME_ZONE)
public static long castToTimeWithTimeZone(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long value)
{
if (session.isLegacyTimestamp()) {
int timeMillis = modulo24Hour(getChronology(session.getTimeZoneKey()), value);
return packDateTimeWithZone(timeMillis, session.getTimeZoneKey());
}
else {
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
// This cast does treat TIMESTAMP as wall time in session TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
return packDateTimeWithZone(localChronology.getZone().convertLocalToUTC(modulo24Hour(value), false), session.getTimeZoneKey());
}
// This cast does treat TIMESTAMP as wall time in session TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
return packDateTimeWithZone(localChronology.getZone().convertLocalToUTC(modulo24Hour(value), false), session.getTimeZoneKey());
}
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE)
public static long castToTimestampWithTimeZone(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long value)
{
if (session.isLegacyTimestamp()) {
return packDateTimeWithZone(value, session.getTimeZoneKey());
}
else {
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
ISOChronology localChronology = getChronology(session.getTimeZoneKey());
// This cast does treat TIMESTAMP as wall time in session TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
return packDateTimeWithZone(localChronology.getZone().convertLocalToUTC(value, false), session.getTimeZoneKey());
}
// This cast does treat TIMESTAMP as wall time in session TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
return packDateTimeWithZone(localChronology.getZone().convertLocalToUTC(value, false), session.getTimeZoneKey());
}
@ScalarOperator(CAST)
@LiteralParameters("x")
@SqlType("varchar(x)")
public static Slice castToSlice(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP) long value)
public static Slice castToSlice(@SqlType(StandardTypes.TIMESTAMP) long value)
{
if (session.isLegacyTimestamp()) {
return utf8Slice(printTimestampWithoutTimeZone(session.getTimeZoneKey(), value));
}
else {
return utf8Slice(printTimestampWithoutTimeZone(value));
}
return utf8Slice(printTimestampWithoutTimeZone(value));
}
@ScalarOperator(CAST)
@LiteralParameters("x")
@SqlType(StandardTypes.TIMESTAMP)
public static long castFromSlice(ConnectorSession session, @SqlType("varchar(x)") Slice value)
public static long castFromSlice(@SqlType("varchar(x)") Slice value)
{
// This accepts value with or without time zone
if (session.isLegacyTimestamp()) {
try {
return parseTimestampWithoutTimeZone(session.getTimeZoneKey(), trim(value).toStringUtf8());
}
catch (IllegalArgumentException e) {
throw new PrestoException(INVALID_CAST_ARGUMENT, "Value cannot be cast to timestamp: " + value.toStringUtf8(), e);
}
try {
return parseTimestampWithoutTimeZone(trim(value).toStringUtf8());
}
else {
try {
return parseTimestampWithoutTimeZone(trim(value).toStringUtf8());
}
catch (IllegalArgumentException e) {
throw new PrestoException(INVALID_CAST_ARGUMENT, "Value cannot be cast to timestamp: " + value.toStringUtf8(), e);
}
catch (IllegalArgumentException e) {
throw new PrestoException(INVALID_CAST_ARGUMENT, "Value cannot be cast to timestamp: " + value.toStringUtf8(), e);
}
}

44
presto-main/src/main/java/io/prestosql/type/TimestampWithTimeZoneOperators.java
View File

@ -141,48 +141,32 @@ public final class TimestampWithTimeZoneOperators
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIME)
public static long castToTime(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long value)
public static long castToTime(@SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long value)
{
if (session.isLegacyTimestamp()) {
return modulo24Hour(unpackChronology(value), unpackMillisUtc(value));
}
else {
return modulo24Hour(castToTimestamp(session, value));
}
return modulo24Hour(castToTimestamp(value));
}
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIME_WITH_TIME_ZONE)
public static long castToTimeWithTimeZone(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long value)
public static long castToTimeWithTimeZone(@SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long value)
{
if (session.isLegacyTimestamp()) {
int millis = modulo24Hour(unpackChronology(value), unpackMillisUtc(value));
return packDateTimeWithZone(millis, unpackZoneKey(value));
}
else {
long millis = modulo24Hour(castToTimestamp(session, value));
ISOChronology localChronology = unpackChronology(value);
long millis = modulo24Hour(castToTimestamp(value));
ISOChronology localChronology = unpackChronology(value);
// This cast does treat TIME as wall time in given TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
// We use value offset in this place to be sure that we will have same hour represented
// in TIME WITH TIME ZONE. Calculating real TZ offset will happen when really required.
// This is done due to inadequate TIME WITH TIME ZONE representation.
return packDateTimeWithZone(millis - localChronology.getZone().getOffset(millis), unpackZoneKey(value));
}
// This cast does treat TIME as wall time in given TZ. This means that in order to get
// its UTC representation we need to shift the value by the offset of TZ.
// We use value offset in this place to be sure that we will have same hour represented
// in TIME WITH TIME ZONE. Calculating real TZ offset will happen when really required.
// This is done due to inadequate TIME WITH TIME ZONE representation.
return packDateTimeWithZone(millis - localChronology.getZone().getOffset(millis), unpackZoneKey(value));
}
@ScalarOperator(CAST)
@SqlType(StandardTypes.TIMESTAMP)
public static long castToTimestamp(ConnectorSession session, @SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long value)
public static long castToTimestamp(@SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long value)
{
if (session.isLegacyTimestamp()) {
return unpackMillisUtc(value);
}
else {
ISOChronology chronology = getChronology(unpackZoneKey(value));
return chronology.getZone().convertUTCToLocal(unpackMillisUtc(value));
}
ISOChronology chronology = getChronology(unpackZoneKey(value));
return chronology.getZone().convertUTCToLocal(unpackMillisUtc(value));
}
@ScalarOperator(CAST)

39
presto-main/src/main/java/io/prestosql/util/JsonUtil.java
View File

@ -27,7 +27,6 @@ import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.BlockBuilder;
import io.prestosql.spi.block.SingleRowBlockWriter;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.type.ArrayType;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.Decimals;
@ -248,7 +247,7 @@ public final class JsonUtil
public interface JsonGeneratorWriter
{
// write a Json value into the JsonGenerator, provided by block and position
void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException;
static JsonGeneratorWriter createJsonGeneratorWriter(Type type)
@ -311,7 +310,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
jsonGenerator.writeNull();
@ -322,7 +321,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -346,7 +345,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -363,7 +362,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -380,7 +379,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -404,7 +403,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -428,7 +427,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -454,7 +453,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -471,7 +470,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -488,7 +487,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -496,7 +495,7 @@ public final class JsonUtil
}
else {
long value = TIMESTAMP.getLong(block, position);
jsonGenerator.writeString(printTimestampWithoutTimeZone(session.getTimeZoneKey(), value));
jsonGenerator.writeString(printTimestampWithoutTimeZone(value));
}
}
}
@ -505,7 +504,7 @@ public final class JsonUtil
implements JsonGeneratorWriter
{
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -531,7 +530,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -541,7 +540,7 @@ public final class JsonUtil
Block arrayBlock = type.getObject(block, position);
jsonGenerator.writeStartArray();
for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
elementWriter.writeJsonValue(jsonGenerator, arrayBlock, i, session);
elementWriter.writeJsonValue(jsonGenerator, arrayBlock, i);
}
jsonGenerator.writeEndArray();
}
@ -563,7 +562,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -580,7 +579,7 @@ public final class JsonUtil
jsonGenerator.writeStartObject();
for (Map.Entry<String, Integer> entry : orderedKeyToValuePosition.entrySet()) {
jsonGenerator.writeFieldName(entry.getKey());
valueWriter.writeJsonValue(jsonGenerator, mapBlock, entry.getValue(), session);
valueWriter.writeJsonValue(jsonGenerator, mapBlock, entry.getValue());
}
jsonGenerator.writeEndObject();
}
@ -600,7 +599,7 @@ public final class JsonUtil
}
@Override
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position, ConnectorSession session)
public void writeJsonValue(JsonGenerator jsonGenerator, Block block, int position)
throws IOException
{
if (block.isNull(position)) {
@ -610,7 +609,7 @@ public final class JsonUtil
Block rowBlock = type.getObject(block, position);
jsonGenerator.writeStartArray();
for (int i = 0; i < rowBlock.getPositionCount(); i++) {
fieldWriters.get(i).writeJsonValue(jsonGenerator, rowBlock, i, session);
fieldWriters.get(i).writeJsonValue(jsonGenerator, rowBlock, i);
}
jsonGenerator.writeEndArray();
}

52
presto-main/src/test/java/io/prestosql/operator/scalar/TestDateTimeFunctions.java
View File

@ -16,20 +16,66 @@ package io.prestosql.operator.scalar;
import io.prestosql.Session;
import io.prestosql.spi.type.TimeType;
import io.prestosql.spi.type.TimeZoneKey;
import io.prestosql.spi.type.TimestampType;
import io.prestosql.testing.TestingSession;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.testng.annotations.Test;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalTime;
import java.time.OffsetTime;
import java.time.ZoneOffset;
import static io.prestosql.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE;
import static io.prestosql.spi.type.TimeZoneKey.UTC_KEY;
import static io.prestosql.spi.type.TimeZoneKey.getTimeZoneKey;
import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
import static io.prestosql.spi.type.VarcharType.createVarcharType;
import static io.prestosql.spi.util.DateTimeZoneIndex.getDateTimeZone;
import static io.prestosql.testing.TestingSession.testSessionBuilder;
public class TestDateTimeFunctions
extends TestDateTimeFunctionsBase
extends AbstractTestFunctions
{
protected static final TimeZoneKey TIME_ZONE_KEY = TestingSession.DEFAULT_TIME_ZONE_KEY;
protected static final DateTimeZone DATE_TIME_ZONE = getDateTimeZone(TIME_ZONE_KEY);
protected static final DateTimeZone UTC_TIME_ZONE = getDateTimeZone(UTC_KEY);
protected static final DateTimeZone DATE_TIME_ZONE_NUMERICAL = getDateTimeZone(getTimeZoneKey("-11:00"));
protected static final TimeZoneKey KATHMANDU_ZONE_KEY = getTimeZoneKey("Asia/Kathmandu");
protected static final DateTimeZone KATHMANDU_ZONE = getDateTimeZone(KATHMANDU_ZONE_KEY);
protected static final ZoneOffset WEIRD_ZONE = ZoneOffset.ofHoursMinutes(7, 9);
protected static final DateTimeZone WEIRD_DATE_TIME_ZONE = DateTimeZone.forID(WEIRD_ZONE.getId());
protected static final DateTime DATE = new DateTime(2001, 8, 22, 0, 0, 0, 0, DateTimeZone.UTC);
protected static final String DATE_LITERAL = "DATE '2001-08-22'";
protected static final String DATE_ISO8601_STRING = "2001-08-22";
protected static final LocalTime TIME = LocalTime.of(3, 4, 5, 321_000_000);
protected static final String TIME_LITERAL = "TIME '03:04:05.321'";
protected static final OffsetTime WEIRD_TIME = OffsetTime.of(3, 4, 5, 321_000_000, WEIRD_ZONE);
protected static final String WEIRD_TIME_LITERAL = "TIME '03:04:05.321 +07:09'";
protected static final DateTime TIMESTAMP = new DateTime(2001, 8, 22, 3, 4, 5, 321, UTC_TIME_ZONE); // This is TIMESTAMP w/o TZ
protected static final DateTime TIMESTAMP_WITH_NUMERICAL_ZONE = new DateTime(2001, 8, 22, 3, 4, 5, 321, DATE_TIME_ZONE_NUMERICAL);
protected static final String TIMESTAMP_LITERAL = "TIMESTAMP '2001-08-22 03:04:05.321'";
protected static final String TIMESTAMP_ISO8601_STRING = "2001-08-22T03:04:05.321-11:00";
protected static final String TIMESTAMP_ISO8601_STRING_NO_TIME_ZONE = "2001-08-22T03:04:05.321";
protected static final DateTime WEIRD_TIMESTAMP = new DateTime(2001, 8, 22, 3, 4, 5, 321, WEIRD_DATE_TIME_ZONE);
protected static final String WEIRD_TIMESTAMP_LITERAL = "TIMESTAMP '2001-08-22 03:04:05.321 +07:09'";
protected static final String WEIRD_TIMESTAMP_ISO8601_STRING = "2001-08-22T03:04:05.321+07:09";
protected static final String INTERVAL_LITERAL = "INTERVAL '90061.234' SECOND";
protected static final Duration DAY_TO_SECOND_INTERVAL = Duration.ofMillis(90061234);
public TestDateTimeFunctions()
{
super(false);
super(testSessionBuilder()
.setTimeZoneKey(TIME_ZONE_KEY)
.setStartTime(Instant.ofEpochMilli(new DateTime(2017, 4, 1, 12, 34, 56, 789, UTC_TIME_ZONE).getMillis()).getEpochSecond())
.build());
}
@Test
@ -67,7 +113,7 @@ public class TestDateTimeFunctions
.setStartTime(new DateTime(2017, 3, 1, 15, 45, 0, 0, KATHMANDU_ZONE).getMillis())
.build();
try (FunctionAssertions localAssertion = new FunctionAssertions(localSession)) {
localAssertion.assertFunctionString("CURRENT_TIME", TIME_WITH_TIME_ZONE, "15:45:00.000 Asia/Kathmandu");
localAssertion.assertFunctionString("CURRENT_TIME", TIME_WITH_TIME_ZONE, "15:30:00.000 Asia/Kathmandu");
}
}

Some files were not shown because too many files have changed in this diff Show More