jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!1335 [I4NBP3] [260] Single Column Read or Update Issue Fix 

Merge pull request !1335 from i-robot/pull261
This commit is contained in:
i-robot 2021-12-22 07:42:16 +00:00 committed by Gitee
parent 79dc5040ad c84c9d2fc2
commit 7d36bfda72
2 changed files with 60 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java
View File

@ -72,6 +72,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -612,6 +613,8 @@ public class OrcPageSourceFactory
*/
private static List<OrcColumn> ensureColumnNameConsistency(List<OrcColumn> fileColumns, List<HiveColumnHandle> desiredColumns)
{
int skipStructColumnCount = 0;
List<Integer> skipStructColumnCountList = new ArrayList<>();
Map<Integer, HiveColumnHandle> desiredColumnsByNumber = desiredColumns.stream().filter(column -> !(column.getHiveColumnIndex() < 0))
.collect(toImmutableMap(HiveColumnHandle::getHiveColumnIndex, identity()));
@ -619,17 +622,16 @@ public class OrcPageSourceFactory
ImmutableList.Builder<OrcColumn> builder = ImmutableList.builderWithExpectedSize(columnCount);
List<Integer> desiredColumnsIndex = desiredColumnsByNumber.keySet().stream().collect(Collectors.toList());
Map<Integer, OrcColumn> fileColumnsByNumber = fileColumns.stream()
.collect(toImmutableMap(orcColumn -> orcColumn.getColumnId().getId(), identity()));
int skipStructColumnCount = 0;
HiveColumnHandle handle = null;
OrcColumn column = null;
Map<Integer, OrcColumn> fileColumnsByNumber = new HashMap<>();
for (OrcColumn orcColumn : fileColumns) {
fileColumnsByNumber.put(orcColumn.getColumnId().getId(), orcColumn);
skipStructColumnCountList.add(skipStructColumnCount);
skipStructColumnCount += getSkipStructColumnCount(orcColumn);
}
for (int index = 0; index < columnCount; index++) {
skipStructColumnCount += handle != null && column != null ? getSkipStructColumnCount(column) : 0;
column = fileColumnsByNumber.get(desiredColumnsIndex.get(index) + SKIP_ORC_FILE_COLUMNS + skipStructColumnCount);
handle = desiredColumnsByNumber.get(desiredColumnsIndex.get(index));
OrcColumn column = fileColumnsByNumber.get(desiredColumnsIndex.get(index) + SKIP_ORC_FILE_COLUMNS + skipStructColumnCountList.get(desiredColumnsIndex.get(index)));
HiveColumnHandle handle = desiredColumnsByNumber.get(desiredColumnsIndex.get(index));
if (handle != null && !column.getColumnName().equals(handle.getName())) {
column = new OrcColumn(column.getPath(), column.getColumnId(), handle.getName(), column.getColumnType(), column.getOrcDataSourceId(), column.getNestedColumns());
}

49
presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java
View File

@ -6888,4 +6888,53 @@ public class TestHiveIntegrationSmokeTest
assertUpdate("DROP TABLE testReadSchema1.testReadStruct6");
assertUpdate("DROP SCHEMA testReadSchema1");
}
@Test
public void testUpdateWithStructDataTypeColumns()
{
assertUpdate("CREATE SCHEMA testReadSchema2");
assertUpdate("CREATE TABLE testReadSchema2.testReadStruct7 (orderkey array(varchar), orderstatus array(row(big int)), lables map(varchar, array(int)), orderstatus1 row(big2 int)) WITH (format='orc',transactional=true)");
assertUpdate("INSERT INTO testReadSchema2.testReadStruct7 VALUES (ARRAY ['YOUNG', 'FASION', 'STYLE'], ARRAY[row(111), row(222)], MAP(ARRAY ['type', 'grand'], ARRAY [ARRAY[1, 2], ARRAY[1,2]]), row(111))", 1);
assertEquals(computeActual("SELECT * FROM testReadSchema2.testReadStruct7").getRowCount(), 1);
assertUpdate("UPDATE testReadSchema2.testReadStruct7 set orderkey = ARRAY['UPDATE','UPDATE','UPDATE']", 1);
assertEquals(computeActual("SELECT * FROM testReadSchema2.testReadStruct7").getRowCount(), 1);
assertUpdate("DROP TABLE testReadSchema2.testReadStruct7");
assertUpdate("DROP SCHEMA testReadSchema2");
}
@Test
public void testReadSingleColumnStructDataTypeColumns()
{
assertUpdate("CREATE SCHEMA testReadSchema3");
assertUpdate("CREATE TABLE testReadSchema3.testReadStruct8 (orderkey array(varchar), orderstatus array(row(big int)), lables map(varchar, array(int)), orderstatus1 row(big2 int)) WITH (format='orc',transactional=true)");
assertUpdate("INSERT INTO testReadSchema3.testReadStruct8 VALUES (ARRAY ['YOUNG', 'FASION', 'STYLE'], ARRAY[row(111), row(222)], MAP(ARRAY ['type', 'grand'], ARRAY [ARRAY[1, 2], ARRAY[1,2]]), row(111))", 1);
assertEquals(computeActual("SELECT orderkey FROM testReadSchema3.testReadStruct8").getRowCount(), 1);
assertEquals(computeActual("SELECT orderstatus FROM testReadSchema3.testReadStruct8").getRowCount(), 1);
assertEquals(computeActual("SELECT lables FROM testReadSchema3.testReadStruct8").getRowCount(), 1);
assertUpdate("DROP TABLE testReadSchema3.testReadStruct8");
assertUpdate("DROP SCHEMA testReadSchema3");
}
@Test
public void testReadFromTablesWithPrimitiveAndStructDataTypeColumns()
{
assertUpdate("CREATE SCHEMA testReadSchema4");
assertUpdate("CREATE TABLE testReadSchema4.testReadStruct9 (id int, orderkey array(varchar), name string, orderstatus array(row(big int)), lables map(varchar, array(int)), orderstatus1 row(big2 int), age int, salary double) WITH (format='orc',transactional=true)");
assertUpdate("INSERT INTO testReadSchema4.testReadStruct9 VALUES (1, ARRAY ['YOUNG', 'FASION', 'STYLE'], 'name1', ARRAY[row(111), row(222)], MAP(ARRAY ['type', 'grand'], ARRAY [ARRAY[1, 2], ARRAY[1,2]]), row(111), 30, 2000.5)", 1);
assertEquals(computeActual("SELECT id FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT name FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT orderkey FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT orderstatus FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT lables FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT age FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT salary FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT id, name, age, salary FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT orderkey, orderstatus, lables, orderstatus1 FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT id, orderkey, name, orderstatus FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT salary, age, lables, orderstatus FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT orderkey, name, lables, orderstatus1 FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertEquals(computeActual("SELECT * FROM testReadSchema4.testReadStruct9").getRowCount(), 1);
assertUpdate("DROP TABLE testReadSchema4.testReadStruct9");
assertUpdate("DROP SCHEMA testReadSchema4");
}
}