From eaaf065220ba158e8dff550fdec49504c485d47c Mon Sep 17 00:00:00 2001 From: chenyidao1 <979136761@qq.com> Date: Thu, 10 Feb 2022 16:35:55 +0800 Subject: [PATCH] fix hetu-core clean code daily check result Signed-off-by: chenyidao1 <979136761@qq.com> --- .../carbondata/CarbondataFileWriter.java | 3 +- .../plugin/carbondata/CarbondataMetadata.java | 33 +++++++++---------- .../plugin/hive/HiveSplitSource.java | 4 +-- .../orc/AbstractOrcRecordReader.java | 8 ++--- .../main/java/io/prestosql/orc/OrcWriter.java | 6 ++-- .../java/io/prestosql/spi/type/Decimals.java | 6 ++-- .../java/io/prestosql/spi/type/RowType.java | 5 +-- .../spi/type/TimeWithTimeZoneType.java | 1 + .../io/prestosql/spi/type/TypeSignature.java | 8 ++--- 9 files changed, 39 insertions(+), 35 deletions(-) diff --git a/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java b/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java index d77b1fdf7..698a4f8e4 100755 --- a/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java +++ b/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java @@ -127,10 +127,11 @@ public class CarbondataFileWriter private boolean isInitDone; private boolean isCommitDone; - public CarbondataFileWriter(Path outPutPath, List inputColumnNames, Properties properties, + public CarbondataFileWriter(Path paramOutPutPath, List inputColumnNames, Properties properties, JobConf configuration, TypeManager typeManager, Optional acidOptions, Optional acidWriteType, OptionalInt taskId) throws SerDeException { + Path outPutPath = paramOutPutPath; this.outPutPath = requireNonNull(outPutPath, "path is null"); // in table creation this can be null if (null != properties.getProperty("location")) { diff --git a/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java b/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java index 818a96f70..ef5f73363 100755 --- a/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java +++ b/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java @@ -935,11 +935,11 @@ public class CarbondataMetadata @Override public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { - SchemaTableName schemaTableName = tableMetadata.getTable(); - String schemaName = schemaTableName.getSchemaName(); - String tableName = schemaTableName.getTableName(); + SchemaTableName localSchemaTableName = tableMetadata.getTable(); + String localSchemaName = localSchemaTableName.getSchemaName(); + String tableName = localSchemaTableName.getTableName(); this.user = session.getUser(); - this.schemaName = schemaName; + this.schemaName = localSchemaName; currentState = State.CREATE_TABLE; List partitionedBy = new ArrayList(); List sortBy = new ArrayList(); @@ -947,7 +947,7 @@ public class CarbondataMetadata Map tableProperties = new HashMap(); getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties); - metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName)); + metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName)); BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties()); // it will get final path to create carbon table @@ -955,10 +955,10 @@ public class CarbondataMetadata Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath(); AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(), - new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString())); + new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString())); hdfsEnvironment.doAs(session.getUser(), () -> { initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration( - new HdfsEnvironment.HdfsContext(session, schemaName, tableName), + new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName), new Path(locationHandle.getJsonSerializableTargetPath()))); CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy, @@ -967,9 +967,9 @@ public class CarbondataMetadata this.tableStorageLocation = Optional.of(targetPath.toString()); try { Map serdeParameters = initSerDeProperties(tableName); - Table table = buildTableObject( + Table localTable = buildTableObject( session.getQueryId(), - schemaName, + localSchemaName, tableName, session.getUser(), columnHandles, @@ -981,11 +981,11 @@ public class CarbondataMetadata true, // carbon table is set as external table prestoVersion, serdeParameters); - PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner()); - HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics(); + PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner()); + HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics(); metastore.createTable( session, - table, + localTable, principalPrivileges, Optional.empty(), ignoreExisting, @@ -1463,11 +1463,10 @@ public class CarbondataMetadata Properties hiveschema = MetastoreUtil.getHiveSchema(table); Configuration configuration = jobContext.getConfiguration(); configuration.set(SET_OVERWRITE, "false"); - CarbonLoadModel carbonLoadModel = - HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration); - LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail(); - carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName()); - CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true); + CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration); + LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail(); + loadModel.setSegmentId(loadMetadataDetails.getLoadName()); + CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true); } catch (IOException e) { LOG.error("Error occurred while committing the insert job.", e); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java index 3b0dd91a4..52d8a8f74 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java @@ -166,7 +166,7 @@ class HiveSplitSource HiveConfig hiveConfig, HiveStorageFormat hiveStorageFormat) { - AtomicReference stateReference = new AtomicReference<>(State.initial()); + AtomicReference localStateReference = new AtomicReference<>(State.initial()); return new HiveSplitSource( session, databaseName, @@ -205,7 +205,7 @@ class HiveSplitSource maxInitialSplits, maxOutstandingSplitsSize, splitLoader, - stateReference, + localStateReference, highMemorySplitSourceCounter, dynamicFilterSupplier, userDefinedCachePredicates, diff --git a/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java b/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java index b6905ca10..967b25b9e 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java @@ -259,9 +259,9 @@ abstract class AbstractOrcRecordReader this.stripes = localStripes.build(); this.stripeFilePositions = localStripeFilePositions.build(); - OrcDataSource orcDataSource = inputOrcDataSource; - orcDataSource = wrapWithCacheIfTinyStripes(orcDataSource, this.stripes, maxMergeDistance, tinyStripeThreshold); - this.orcDataSource = orcDataSource; + OrcDataSource localOrcDataSource = inputOrcDataSource; + localOrcDataSource = wrapWithCacheIfTinyStripes(localOrcDataSource, this.stripes, maxMergeDistance, tinyStripeThreshold); + this.orcDataSource = localOrcDataSource; this.splitLength = splitLength; this.fileRowCount = stripeInfos.stream() @@ -279,7 +279,7 @@ abstract class AbstractOrcRecordReader // their constructors is confusing. AggregatedMemoryContext streamReadersSystemMemoryContext = this.systemMemoryUsage.newAggregatedMemoryContext(); stripeReader = new StripeReader( - orcDataSource, + localOrcDataSource, legacyFileTimeZone.toTimeZone().toZoneId(), decompressor, orcTypes, diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java b/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java index 16c3149e3..f5c0e42b8 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java +++ b/presto-orc/src/main/java/io/prestosql/orc/OrcWriter.java @@ -168,13 +168,13 @@ public final class OrcWriter // create column writers OrcType localRootType = orcTypes.get(ROOT_COLUMN); checkArgument(localRootType.getFieldCount() == types.size()); - ImmutableList.Builder columnWriters = ImmutableList.builder(); + ImmutableList.Builder localColumnWriters = ImmutableList.builder(); ImmutableSet.Builder sliceColumnWriters = ImmutableSet.builder(); for (int fieldId = 0; fieldId < types.size(); fieldId++) { OrcColumnId fieldColumnIndex = localRootType.getFieldTypeIndex(fieldId); Type fieldType = types.get(fieldId); ColumnWriter columnWriter = createColumnWriter(fieldColumnIndex, orcTypes, fieldType, compression, maxCompressionBufferSize, options.getMaxStringStatisticsLimit()); - columnWriters.add(columnWriter); + localColumnWriters.add(columnWriter); if (columnWriter instanceof SliceDictionaryColumnWriter) { sliceColumnWriters.add((SliceDictionaryColumnWriter) columnWriter); @@ -187,7 +187,7 @@ public final class OrcWriter } } } - this.columnWriters = columnWriters.build(); + this.columnWriters = localColumnWriters.build(); this.dictionaryCompressionOptimizer = new DictionaryCompressionOptimizer( sliceColumnWriters.build(), stripeMinBytes, diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/Decimals.java b/presto-spi/src/main/java/io/prestosql/spi/type/Decimals.java index d82ede4bc..4eeb47fe9 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/Decimals.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/Decimals.java @@ -196,8 +196,9 @@ public final class Decimals return toString(unscaledValue.toString(), scale); } - private static String toString(String unscaledValueString, int scale) + private static String toString(String inputString, int scale) { + String unscaledValueString = inputString; StringBuilder resultBuilder = new StringBuilder(); // add sign if (unscaledValueString.startsWith("-")) { @@ -274,8 +275,9 @@ public final class Decimals decimalType.writeSlice(blockBuilder, encodeScaledValue(value)); } - public static BigDecimal rescale(BigDecimal value, DecimalType type) + public static BigDecimal rescale(BigDecimal inputValue, DecimalType type) { + BigDecimal value = inputValue; value = value.setScale(type.getScale(), UNNECESSARY); if (value.precision() > type.getPrecision()) { diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/RowType.java b/presto-spi/src/main/java/io/prestosql/spi/type/RowType.java index ecaf16d78..9400aacae 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/RowType.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/RowType.java @@ -56,11 +56,11 @@ public class RowType public static RowType anonymous(List types) { - List fields = types.stream() + List localFields = types.stream() .map(type -> new Field(Optional.empty(), type)) .collect(Collectors.toList()); - return new RowType(makeSignature(fields), fields); + return new RowType(makeSignature(localFields), localFields); } // Only RowParametricType.createType should call this method @@ -212,6 +212,7 @@ public class RowType return fields.stream().allMatch(field -> field.getType().isOrderable()); } + @Override public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) { //FIXME: KEN: Generic should be used diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/TimeWithTimeZoneType.java b/presto-spi/src/main/java/io/prestosql/spi/type/TimeWithTimeZoneType.java index 349be49e9..b689ee096 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/TimeWithTimeZoneType.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/TimeWithTimeZoneType.java @@ -47,6 +47,7 @@ public final class TimeWithTimeZoneType return leftValue == rightValue; } + @Override public long hash(Block block, int position) { return AbstractLongType.hash(unpackMillisUtc(block.getLong(position, 0))); diff --git a/presto-spi/src/main/java/io/prestosql/spi/type/TypeSignature.java b/presto-spi/src/main/java/io/prestosql/spi/type/TypeSignature.java index 98e005866..a5d2db144 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/type/TypeSignature.java +++ b/presto-spi/src/main/java/io/prestosql/spi/type/TypeSignature.java @@ -114,7 +114,7 @@ public class TypeSignature } String baseName = null; - List parameters = new ArrayList<>(); + List localParameters = new ArrayList<>(); int parameterStart = -1; int bracketCount = 0; @@ -138,17 +138,17 @@ public class TypeSignature checkArgument(bracketCount >= 0, "Bad type signature: '%s'", signature); if (bracketCount == 0) { checkArgument(parameterStart >= 0, "Bad type signature: '%s'", signature); - parameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters)); + localParameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters)); parameterStart = i + 1; if (i == signature.length() - 1) { - return new TypeSignature(baseName, parameters); + return new TypeSignature(baseName, localParameters); } } } else if (c == ',') { if (bracketCount == 1) { checkArgument(parameterStart >= 0, "Bad type signature: '%s'", signature); - parameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters)); + localParameters.add(parseTypeSignatureParameter(signature, parameterStart, i, literalCalculationParameters)); parameterStart = i + 1; } }