jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix hetu-core clean code daily check result 

Signed-off-by: chenyidao1 <979136761@qq.com>
This commit is contained in:
chenyidao1 2022-02-10 16:35:55 +08:00
parent 8152686fa1
commit eaaf065220
9 changed files with 39 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java
View File

@ -127,10 +127,11 @@ public class CarbondataFileWriter
private boolean isInitDone;
private boolean isCommitDone;
public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
{
Path outPutPath = paramOutPutPath;
this.outPutPath = requireNonNull(outPutPath, "path is null");
// in table creation this can be null
if (null != properties.getProperty("location")) {

33
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java
View File

@ -935,11 +935,11 @@ public class CarbondataMetadata
@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();
String tableName = schemaTableName.getTableName();
SchemaTableName localSchemaTableName = tableMetadata.getTable();
String localSchemaName = localSchemaTableName.getSchemaName();
String tableName = localSchemaTableName.getTableName();
this.user = session.getUser();
this.schemaName = schemaName;
this.schemaName = localSchemaName;
currentState = State.CREATE_TABLE;
List<String> partitionedBy = new ArrayList<String>();
List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
@ -947,7 +947,7 @@ public class CarbondataMetadata
Map<String, String> tableProperties = new HashMap<String, String>();
getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
// it will get final path to create carbon table
@ -955,10 +955,10 @@ public class CarbondataMetadata
Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
hdfsEnvironment.doAs(session.getUser(), () -> {
initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName),
new Path(locationHandle.getJsonSerializableTargetPath())));
CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
@ -967,9 +967,9 @@ public class CarbondataMetadata
this.tableStorageLocation = Optional.of(targetPath.toString());
try {
Map<String, String> serdeParameters = initSerDeProperties(tableName);
Table table = buildTableObject(
Table localTable = buildTableObject(
session.getQueryId(),
schemaName,
localSchemaName,
tableName,
session.getUser(),
columnHandles,
@ -981,11 +981,11 @@ public class CarbondataMetadata
true, // carbon table is set as external table
prestoVersion,
serdeParameters);
PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner());
HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
metastore.createTable(
session,
table,
localTable,
principalPrivileges,
Optional.empty(),
ignoreExisting,
@ -1463,11 +1463,10 @@ public class CarbondataMetadata
Properties hiveschema = MetastoreUtil.getHiveSchema(table);
Configuration configuration = jobContext.getConfiguration();
configuration.set(SET_OVERWRITE, "false");
CarbonLoadModel carbonLoadModel =
HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail();
carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName());
CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true);
CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail();
loadModel.setSegmentId(loadMetadataDetails.getLoadName());
CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true);
}
catch (IOException e) {
LOG.error("Error occurred while committing the insert job.", e);

4
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java
View File

@ -166,7 +166,7 @@ class HiveSplitSource
HiveConfig hiveConfig,
HiveStorageFormat hiveStorageFormat)
{
AtomicReference<State> stateReference = new AtomicReference<>(State.initial());
AtomicReference<State> localStateReference = new AtomicReference<>(State.initial());
return new HiveSplitSource(
session,
databaseName,
@ -205,7 +205,7 @@ class HiveSplitSource
maxInitialSplits,
maxOutstandingSplitsSize,
splitLoader,
stateReference,
localStateReference,
highMemorySplitSourceCounter,
dynamicFilterSupplier,
userDefinedCachePredicates,

8
presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java
View File

@ -259,9 +259,9 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
this.stripes = localStripes.build();
this.stripeFilePositions = localStripeFilePositions.build();
OrcDataSource orcDataSource = inputOrcDataSource;
orcDataSource = wrapWithCacheIfTinyStripes(orcDataSource, this.stripes, maxMergeDistance, tinyStripeThreshold);
this.orcDataSource = orcDataSource;
OrcDataSource localOrcDataSource = inputOrcDataSource;
localOrcDataSource = wrapWithCacheIfTinyStripes(localOrcDataSource, this.stripes, maxMergeDistance, tinyStripeThreshold);
this.orcDataSource = localOrcDataSource;
this.splitLength = splitLength;
this.fileRowCount = stripeInfos.stream()
@ -279,7 +279,7 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
// their constructors is confusing.
AggregatedMemoryContext streamReadersSystemMemoryContext = this.systemMemoryUsage.newAggregatedMemoryContext();
stripeReader = new StripeReader(
orcDataSource,
localOrcDataSource,
legacyFileTimeZone.toTimeZone().toZoneId(),
decompressor,
orcTypes,

6
presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java
View File

@ -168,13 +168,13 @@ public final class OrcWriter
// create column writers
OrcType localRootType = orcTypes.get(ROOT_COLUMN);
checkArgument(localRootType.getFieldCount() == types.size());
ImmutableList.Builder<ColumnWriter> columnWriters = ImmutableList.builder();
ImmutableList.Builder<ColumnWriter> localColumnWriters = ImmutableList.builder();
ImmutableSet.Builder<SliceDictionaryColumnWriter> sliceColumnWriters = ImmutableSet.builder();
for (int fieldId = 0; fieldId < types.size(); fieldId++) {
OrcColumnId fieldColumnIndex = localRootType.getFieldTypeIndex(fieldId);
Type fieldType = types.get(fieldId);
ColumnWriter columnWriter = createColumnWriter(fieldColumnIndex, orcTypes, fieldType, compression, maxCompressionBufferSize, options.getMaxStringStatisticsLimit());
columnWriters.add(columnWriter);
localColumnWriters.add(columnWriter);
if (columnWriter instanceof SliceDictionaryColumnWriter) {
sliceColumnWriters.add((SliceDictionaryColumnWriter) columnWriter);
@ -187,7 +187,7 @@ public final class OrcWriter
}
}
}
this.columnWriters = columnWriters.build();
this.columnWriters = localColumnWriters.build();
this.dictionaryCompressionOptimizer = new DictionaryCompressionOptimizer(
sliceColumnWriters.build(),
stripeMinBytes,

6
presto-spi/src/main/java/io/prestosql/spi/type/Decimals.java
View File

@ -196,8 +196,9 @@ public final class Decimals
return toString(unscaledValue.toString(), scale);
}
private static String toString(String unscaledValueString, int scale)
private static String toString(String inputString, int scale)
{
String unscaledValueString = inputString;
StringBuilder resultBuilder = new StringBuilder();
// add sign
if (unscaledValueString.startsWith("-")) {
@ -274,8 +275,9 @@ public final class Decimals
decimalType.writeSlice(blockBuilder, encodeScaledValue(value));
}
public static BigDecimal rescale(BigDecimal value, DecimalType type)
public static BigDecimal rescale(BigDecimal inputValue, DecimalType type)
{
BigDecimal value = inputValue;
value = value.setScale(type.getScale(), UNNECESSARY);
if (value.precision() > type.getPrecision()) {

5
presto-spi/src/main/java/io/prestosql/spi/type/RowType.java
View File

@ -56,11 +56,11 @@ public class RowType
public static RowType anonymous(List<Type> types)
{
List<Field> fields = types.stream()
List<Field> localFields = types.stream()
.map(type -> new Field(Optional.empty(), type))
.collect(Collectors.toList());
return new RowType(makeSignature(fields), fields);
return new RowType(makeSignature(localFields), localFields);
}
// Only RowParametricType.createType should call this method
@ -212,6 +212,7 @@ public class RowType
return fields.stream().allMatch(field -> field.getType().isOrderable());
}
@Override
public <T> boolean equalTo(Block<T> leftBlock, int leftPosition, Block<T> rightBlock, int rightPosition)
{
//FIXME: KEN: Generic should be used

1
presto-spi/src/main/java/io/prestosql/spi/type/TimeWithTimeZoneType.java
View File

@ -47,6 +47,7 @@ public final class TimeWithTimeZoneType
return leftValue == rightValue;
}
@Override
public long hash(Block block, int position)
{
return AbstractLongType.hash(unpackMillisUtc(block.getLong(position, 0)));

8
presto-spi/src/main/java/io/prestosql/spi/type/TypeSignature.java
View File

@ -114,7 +114,7 @@ public class TypeSignature
}
String baseName = null;
List<TypeSignatureParameter> parameters = new ArrayList<>();
List<TypeSignatureParameter> localParameters = new ArrayList<>();
int parameterStart = -1;
int bracketCount = 0;
@ -138,17 +138,17 @@ public class TypeSignature
checkArgument(bracketCount >= 0, "Bad type signature: '%s'", signature);
if (bracketCount == 0) {
checkArgument(parameterStart >= 0, "Bad type signature: '%s'", signature);
parameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters));
localParameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters));
parameterStart = i + 1;
if (i == signature.length() - 1) {
return new TypeSignature(baseName, parameters);
return new TypeSignature(baseName, localParameters);
}
}
}
else if (c == ',') {
if (bracketCount == 1) {
checkArgument(parameterStart >= 0, "Bad type signature: '%s'", signature);
parameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters));
localParameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters));
parameterStart = i + 1;
}
}