jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added hIndex, DataCache support in pushdown 

added plan printer for pushdown predicates; and rebased code
revert to generic block + fix ut failures for partition delete
This commit is contained in:
nitin.kashyap 2020-08-24 21:34:58 +05:30
parent 37930273a4
commit d35aa29ef2
86 changed files with 1535 additions and 493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

252
presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt
View File

@ -22,19 +22,37 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_29"])
single aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_42", "c_customer_id_29", "c_email_address_44", "c_first_name_36", "c_last_name_37", "c_login_43", "c_preferred_cust_flag_38", "d_year_52"])
partial aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_28"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["cs_sold_date_sk"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_46"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_123"])
single aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_136", "c_customer_id_123", "c_email_address_138", "c_first_name_130", "c_last_name_131", "c_login_137", "c_preferred_cust_flag_132", "d_year_146"])
partial aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_122"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_140"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_247"])
final aggregation over (c_birth_country_260, c_customer_id_247, c_email_address_262, c_first_name_254, c_last_name_255, c_login_261, c_preferred_cust_flag_256, d_year_293)
@ -51,27 +69,54 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_354"])
single aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_367", "c_customer_id_354", "c_email_address_369", "c_first_name_361", "c_last_name_362", "c_login_368", "c_preferred_cust_flag_363", "d_year_411"])
partial aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_353"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk_374"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["cs_sold_date_sk_371"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_405"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_482"])
single aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_495", "c_customer_id_482", "c_email_address_497", "c_first_name_489", "c_last_name_490", "c_login_496", "c_preferred_cust_flag_491", "d_year_539"])
partial aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_481"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_503"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_499"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_533"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_640"])
single aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_653", "c_customer_id_640", "c_email_address_655", "c_first_name_647", "c_last_name_648", "c_login_654", "c_preferred_cust_flag_649", "d_year_686"])
partial aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_639"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_660"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_657"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_680"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_747"])
final aggregation over (c_birth_country_760, c_customer_id_747, c_email_address_762, c_first_name_754, c_last_name_755, c_login_761, c_preferred_cust_flag_756, d_year_804)
local exchange (GATHER, SINGLE, [])
@ -87,20 +132,38 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_875"])
single aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_888", "c_customer_id_875", "c_email_address_890", "c_first_name_882", "c_last_name_883", "c_login_889", "c_preferred_cust_flag_884", "d_year_932"])
partial aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_874"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_896"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_892"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_926"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_1033"])
single aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_1046", "c_customer_id_1033", "c_email_address_1048", "c_first_name_1040", "c_last_name_1041", "c_login_1047", "c_preferred_cust_flag_1042", "d_year_1079"])
partial aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_1032"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_1053"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_1050"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_1073"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_1140"])
final aggregation over (c_birth_country_1153, c_customer_id_1140, c_email_address_1155, c_first_name_1147, c_last_name_1148, c_login_1154, c_preferred_cust_flag_1149, d_year_1197)
local exchange (GATHER, SINGLE, [])
@ -116,27 +179,54 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_1268"])
single aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_1281", "c_customer_id_1268", "c_email_address_1283", "c_first_name_1275", "c_last_name_1276", "c_login_1282", "c_preferred_cust_flag_1277", "d_year_1325"])
partial aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_1267"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_1289"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_1285"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_1319"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_1426"])
single aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_1439", "c_customer_id_1426", "c_email_address_1441", "c_first_name_1433", "c_last_name_1434", "c_login_1440", "c_preferred_cust_flag_1435", "d_year_1472"])
partial aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_1425"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_1446"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_1443"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_1466"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_1533"])
single aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_1546", "c_customer_id_1533", "c_email_address_1548", "c_first_name_1540", "c_last_name_1541", "c_login_1547", "c_preferred_cust_flag_1542", "d_year_1590"])
partial aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_1532"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk_1553"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["cs_sold_date_sk_1550"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_1584"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_1661"])
final aggregation over (c_birth_country_1674, c_customer_id_1661, c_email_address_1676, c_first_name_1668, c_last_name_1669, c_login_1675, c_preferred_cust_flag_1670, d_year_1718)
local exchange (GATHER, SINGLE, [])
@ -154,19 +244,37 @@ local exchange (GATHER, SINGLE, [])
scan customer
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_1819"])
single aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_1832", "c_customer_id_1819", "c_email_address_1834", "c_first_name_1826", "c_last_name_1827", "c_login_1833", "c_preferred_cust_flag_1828", "d_year_1865"])
partial aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_1818"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_1839"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_1836"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_1859"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_1926"])
single aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_1939", "c_customer_id_1926", "c_email_address_1941", "c_first_name_1933", "c_last_name_1934", "c_login_1940", "c_preferred_cust_flag_1935", "d_year_1983"])
partial aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_1925"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk_1946"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["cs_sold_date_sk_1943"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_1977"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_2054"])
final aggregation over (c_birth_country_2067, c_customer_id_2054, c_email_address_2069, c_first_name_2061, c_last_name_2062, c_login_2068, c_preferred_cust_flag_2063, d_year_2111)
local exchange (GATHER, SINGLE, [])

84
presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt
View File

@ -19,12 +19,21 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_29"])
single aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_42", "c_customer_id_29", "c_email_address_44", "c_first_name_36", "c_last_name_37", "c_login_43", "c_preferred_cust_flag_38", "d_year_52"])
partial aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_28"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_46"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_153"])
final aggregation over (c_birth_country_166, c_customer_id_153, c_email_address_168, c_first_name_160, c_last_name_161, c_login_167, c_preferred_cust_flag_162, d_year_199)
@ -41,20 +50,38 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_260"])
single aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_273", "c_customer_id_260", "c_email_address_275", "c_first_name_267", "c_last_name_268", "c_login_274", "c_preferred_cust_flag_269", "d_year_317"])
partial aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_259"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_281"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_277"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_311"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_418"])
single aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_431", "c_customer_id_418", "c_email_address_433", "c_first_name_425", "c_last_name_426", "c_login_432", "c_preferred_cust_flag_427", "d_year_464"])
partial aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_417"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_438"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_435"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_458"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_525"])
final aggregation over (c_birth_country_538, c_customer_id_525, c_email_address_540, c_first_name_532, c_last_name_533, c_login_539, c_preferred_cust_flag_534, d_year_582)
local exchange (GATHER, SINGLE, [])
@ -72,12 +99,21 @@ local exchange (GATHER, SINGLE, [])
scan customer
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_683"])
single aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_birth_country_696", "c_customer_id_683", "c_email_address_698", "c_first_name_690", "c_last_name_691", "c_login_697", "c_preferred_cust_flag_692", "d_year_729"])
partial aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_682"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_703"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_700"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_723"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_790"])
final aggregation over (c_birth_country_803, c_customer_id_790, c_email_address_805, c_first_name_797, c_last_name_798, c_login_804, c_preferred_cust_flag_799, d_year_847)
local exchange (GATHER, SINGLE, [])

4
presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt
View File

@ -16,10 +16,10 @@ remote exchange (GATHER, SINGLE, [])
scan store
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan date_dim
scan household_demographics
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan household_demographics
scan date_dim
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_sk"])
scan customer

4
presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt
View File

@ -16,10 +16,10 @@ remote exchange (GATHER, SINGLE, [])
scan date_dim
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan store
scan household_demographics
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan household_demographics
scan store
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_sk"])
scan customer

84
presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt
View File

@ -19,12 +19,21 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_17"])
single aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_17", "c_first_name_24", "c_last_name_25", "d_year_40"])
partial aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_16"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_34"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_109"])
final aggregation over (c_customer_id_109, c_first_name_116, c_last_name_117, d_year_155)
@ -41,20 +50,38 @@ local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer
remote exchange (REPARTITION, HASH, ["c_customer_id_200"])
single aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_200", "c_first_name_207", "c_last_name_208", "d_year_257"])
partial aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_199"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_221"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_217"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_251"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_326"])
single aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_326", "c_first_name_333", "c_last_name_334", "d_year_372"])
partial aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_325"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_346"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_343"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_366"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_417"])
final aggregation over (c_customer_id_417, c_first_name_424, c_last_name_425, d_year_474)
local exchange (GATHER, SINGLE, [])
@ -72,12 +99,21 @@ local exchange (GATHER, SINGLE, [])
scan customer
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_543"])
single aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589)
join (INNER, REPLICATED):
join (INNER, REPLICATED):
values (0 rows)
values (0 rows)
values (0 rows)
final aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["c_customer_id_543", "c_first_name_550", "c_last_name_551", "d_year_589"])
partial aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589)
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["c_customer_sk_542"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["ss_customer_sk_563"])
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_560"])
values (0 rows)
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, ["d_date_sk_583"])
values (0 rows)
remote exchange (REPARTITION, HASH, ["c_customer_id_634"])
final aggregation over (c_customer_id_634, c_first_name_641, c_last_name_642, d_year_691)
local exchange (GATHER, SINGLE, [])

9
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java
View File

@ -1976,7 +1976,11 @@ public class HiveMetadata
.orElseThrow(() -> new TableNotFoundException(hiveTable.getSchemaTableName()));
if (hmsTable.getStorage().getStorageFormat().getSerDe().equalsIgnoreCase(HiveStorageFormat.ORC.getSerDe())) {
predicate = hiveTable.getEnforcedConstraint();
ImmutableMap.Builder<ColumnHandle, Domain> pushedDown = ImmutableMap.builder();
pushedDown.putAll(hiveTable.getCompactEffectivePredicate().getDomains().get().entrySet().stream()
.collect(toMap(e -> (ColumnHandle) e.getKey(), e -> e.getValue())));
predicate = predicate.intersect(withColumnDomains(pushedDown.build()));
}
}
@ -2059,7 +2063,8 @@ public class HiveMetadata
.forEach(predicateColumnNames::add);
List<TupleDomain<HiveColumnHandle>> newEffectivePredicates = null;
if (HiveSessionProperties.isOrcPredicatePushdownEnabled(session)) {
if (HiveSessionProperties.isOrcPredicatePushdownEnabled(session)
&& HiveSessionProperties.isOrcDisjunctPredicatePushdownEnabled(session)) {
newEffectivePredicates = builder.build();
newEffectivePredicates.stream().forEach(nfp ->

19
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java
View File

@ -41,6 +41,7 @@ import org.joda.time.DateTimeZone;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -132,12 +133,20 @@ public class HivePageSourceProvider
Configuration configuration = hdfsEnvironment.getConfiguration(
new HdfsEnvironment.HdfsContext(session, hiveSplit.getDatabase(), hiveSplit.getTable()), path);
List<IndexMetadata> indexes = null;
List<IndexMetadata> indexes = new ArrayList<>();
if (indexCache != null && session.isHeuristicIndexFilterEnabled()) {
indexes = indexCache.getIndices(
session.getCatalog().orElse(null),
hiveTable.getSchemaTableName().toString(), hiveSplit, hiveTable.getCompactEffectivePredicate(),
hiveTable.getPartitionColumns());
indexes.addAll(this.indexCache.getIndices(session
.getCatalog().orElse(null), hiveTable
.getSchemaTableName().toString(), hiveSplit, hiveTable.getCompactEffectivePredicate(),
hiveTable.getPartitionColumns()));
if (hiveTable.getAdditionalCompactEffectivePredicate().isPresent() && hiveTable.getAdditionalCompactEffectivePredicate().get().size() > 0) {
hiveTable.getAdditionalCompactEffectivePredicate().get().forEach(orPredicate ->
indexes.addAll(this.indexCache.getIndices(session
.getCatalog().orElse(null), hiveTable
.getSchemaTableName().toString(), hiveSplit, orPredicate, hiveTable
.getPartitionColumns())));
}
}
Optional<List<IndexMetadata>> indexOptional =
indexes == null || indexes.isEmpty() ? Optional.empty() : Optional.of(indexes);

8
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSelectivePageSourceFactory.java
View File

@ -13,10 +13,9 @@
*/
package io.prestosql.plugin.hive;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.plugin.hive.coercions.HiveCoercer;
import io.prestosql.spi.connector.ConnectorPageSource;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.dynamicfilter.DynamicFilter;
import io.prestosql.spi.heuristicindex.IndexMetadata;
import io.prestosql.spi.predicate.TupleDomain;
import org.apache.hadoop.conf.Configuration;
@ -44,9 +43,10 @@ public interface HiveSelectivePageSourceFactory
TupleDomain<HiveColumnHandle> domainPredicate,
Optional<List<TupleDomain<HiveColumnHandle>>> additionPredicates,
DateTimeZone hiveStorageTimeZone,
Map<ColumnHandle, DynamicFilter> dynamicFilter,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
Optional<List<IndexMetadata>> indexes,
boolean splitCacheable);
boolean splitCacheable,
List<HivePageSourceProvider.ColumnMapping> columnMappings,
Map<Integer, HiveCoercer> coercers);
}

26
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java
View File

@ -85,8 +85,10 @@ public final class HiveSessionProperties
private static final String TEMPORARY_STAGING_DIRECTORY_ENABLED = "temporary_staging_directory_enabled";
private static final String TEMPORARY_STAGING_DIRECTORY_PATH = "temporary_staging_directory_path";
private static final String DYNAMIC_FILTERING_SPLIT_FILTERING = "dynamic_filtering_partition_filtering";
private static final String ORC_PREDICATE_PUSHDOWN = "orc_predicate_pushdown_enabled";
private static final String DYNAMIC_FILTERING_ROW_FILTERING_THRESHOLD = "dynamic_filtering_filter_rows_threshold";
private static final String ORC_PREDICATE_PUSHDOWN = "orc_predicate_pushdown_enabled";
private static final String ORC_DISJUCT_PREDICATE_PUSHDOWN = "orc_disjunct_predicate_pushdown_enabled";
private static final String ORC_PUSHDOWN_DATACACHE = "orc_pushdown_data_cache_enabled";
private final List<PropertyMetadata<?>> sessionProperties;
@ -365,8 +367,18 @@ public final class HiveSessionProperties
false),
booleanProperty(
ORC_PREDICATE_PUSHDOWN,
"Experimental: Consume deterministic predicates for ORC scan.",
"Experimental: Consume deterministic predicates(conjucts: AND) for ORC scan.",
false,
false),
booleanProperty(
ORC_DISJUCT_PREDICATE_PUSHDOWN,
"Experimental: Consume deterministic predicates(disjucts: OR) for ORC scan.",
true,
false),
booleanProperty(
ORC_PUSHDOWN_DATACACHE,
"Experimental: Enable data cache or result cache with predicate pushdown.",
true,
false));
}
@ -621,4 +633,14 @@ public final class HiveSessionProperties
{
return session.getProperty(ORC_PREDICATE_PUSHDOWN, Boolean.class);
}
public static boolean isOrcDisjunctPredicatePushdownEnabled(ConnectorSession session)
{
return session.getProperty(ORC_DISJUCT_PREDICATE_PUSHDOWN, Boolean.class);
}
public static boolean isOrcPushdownDataCacheEnabled(ConnectorSession session)
{
return session.getProperty(ORC_PUSHDOWN_DATACACHE, Boolean.class);
}
}

1
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java
View File

@ -134,6 +134,7 @@ class HiveSplitSource
this.maxSplitSize = getMaxSplitSize(session);
this.maxInitialSplitSize = getMaxInitialSplitSize(session);
this.remainingInitialSplits = new AtomicInteger(maxInitialSplits);
this.dynamicFilterSupplier = dynamicFilterSupplier;
this.isSplitFilteringEnabled = isDynamicFilteringSplitFilteringEnabled(session);
this.userDefinedCachePredicates = userDefinedCachedPredicates;

22
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveTableHandle.java
View File

@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ConnectorTableHandle;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.predicate.Domain;
import io.prestosql.spi.predicate.TupleDomain;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@ -28,6 +29,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
@ -274,6 +277,25 @@ public class HiveTableHandle
&& additionalCompactEffectivePredicate.get().size() > 0;
}
private String formatPredicate(Function<Domain, String> printer, TupleDomain<HiveColumnHandle> predicate)
{
return predicate.getDomains().get().entrySet().stream()
.map(filter -> filter.getKey().getColumnName() + " <- " + printer.apply(filter.getValue()))
.collect(Collectors.joining(" AND ", "{", "}"));
}
@Override
public String getAdditionalFilterConditions(Function<Domain, String> printer)
{
if (additionalCompactEffectivePredicate.isPresent()) {
return additionalCompactEffectivePredicate.get().stream()
.map(predicate -> "[ " + formatPredicate(printer, predicate) + " ]")
.collect(Collectors.joining(" OR "));
}
return "";
}
@Override
public boolean equals(Object o)
{

4
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriteUtils.java
View File

@ -326,7 +326,7 @@ public final class HiveWriteUtils
if (isArrayType(type)) {
Type elementType = type.getTypeParameters().get(0);
Block<T> arrayBlock = block.getObject(position, Block.class);
Block arrayBlock = block.getObject(position, Block.class);
List<Object> list = new ArrayList<>(arrayBlock.getPositionCount());
for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
@ -1093,7 +1093,7 @@ public final class HiveWriteUtils
@Override
public <T> void setField(Block<T> block, int position)
{
Block mapBlock = block.getObject(position, Block.class);
Block<T> mapBlock = block.getObject(position, Block.class);
Map<Object, Object> map = new HashMap<>(mapBlock.getPositionCount() * 2);
for (int i = 0; i < mapBlock.getPositionCount(); i += 2) {
Object key = getField(keyType, mapBlock, i);

12
presto-hive/src/main/java/io/prestosql/plugin/hive/coercions/DecimalCoercers.java
View File

@ -21,8 +21,6 @@ import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.DoubleType;
import io.prestosql.spi.type.RealType;
import java.util.function.Function;
import static io.prestosql.spi.type.DecimalConversions.doubleToLongDecimal;
import static io.prestosql.spi.type.DecimalConversions.doubleToShortDecimal;
import static io.prestosql.spi.type.DecimalConversions.longDecimalToDouble;
@ -43,7 +41,7 @@ public final class DecimalCoercers
{
private DecimalCoercers() {}
public static Function<Block, Block> createDecimalToDecimalCoercer(DecimalType fromType, DecimalType toType)
public static HiveCoercer createDecimalToDecimalCoercer(DecimalType fromType, DecimalType toType)
{
if (fromType.isShort()) {
if (toType.isShort()) {
@ -148,7 +146,7 @@ public final class DecimalCoercers
}
}
public static Function<Block, Block> createDecimalToDoubleCoercer(DecimalType fromType)
public static HiveCoercer createDecimalToDoubleCoercer(DecimalType fromType)
{
if (fromType.isShort()) {
return new ShortDecimalToDoubleCoercer(fromType);
@ -193,7 +191,7 @@ public final class DecimalCoercers
}
}
public static Function<Block, Block> createDecimalToRealCoercer(DecimalType fromType)
public static HiveCoercer createDecimalToRealCoercer(DecimalType fromType)
{
if (fromType.isShort()) {
return new ShortDecimalToRealCoercer(fromType);
@ -238,7 +236,7 @@ public final class DecimalCoercers
}
}
public static Function<Block, Block> createDoubleToDecimalCoercer(DecimalType toType)
public static HiveCoercer createDoubleToDecimalCoercer(DecimalType toType)
{
if (toType.isShort()) {
return new DoubleToShortDecimalCoercer(toType);
@ -280,7 +278,7 @@ public final class DecimalCoercers
}
}
public static Function<Block, Block> createRealToDecimalCoercer(DecimalType toType)
public static HiveCoercer createRealToDecimalCoercer(DecimalType toType)
{
if (toType.isShort()) {
return new RealToShortDecimalCoercer(toType);

24
presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSource.java
View File

@ -18,12 +18,18 @@ import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.OrcDataSource;
import io.prestosql.orc.OrcSelectiveRecordReader;
import io.prestosql.plugin.hive.FileFormatDataSourceStats;
import io.prestosql.plugin.hive.HiveColumnHandle;
import io.prestosql.plugin.hive.HivePageSourceProvider.ColumnMapping;
import io.prestosql.spi.Page;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ConnectorPageSource;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.TypeManager;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.List;
import static com.google.common.base.MoreObjects.toStringHelper;
import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_BAD_DATA;
@ -40,17 +46,31 @@ public class OrcSelectivePageSource
private final FileFormatDataSourceStats stats;
private boolean closed;
private final Type[] types;
public OrcSelectivePageSource(
OrcSelectiveRecordReader recordReader,
OrcDataSource orcDataSource,
AggregatedMemoryContext systemMemoryContext,
FileFormatDataSourceStats stats)
FileFormatDataSourceStats stats,
ConnectorSession session,
List<ColumnMapping> columnMappings,
TypeManager typeManager)
{
this.recordReader = requireNonNull(recordReader, "recordReader is null");
this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null");
this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null");
this.stats = requireNonNull(stats, "stats is null");
types = new Type[columnMappings.size()];
for (int columnIndex = 0; columnIndex < columnMappings.size(); columnIndex++) {
ColumnMapping columnMapping = columnMappings.get(columnIndex);
HiveColumnHandle column = columnMapping.getHiveColumnHandle();
String name = column.getName();
Type type = typeManager.getType(column.getTypeSignature());
types[columnIndex] = type;
}
}
@Override
@ -78,7 +98,9 @@ public class OrcSelectivePageSource
Page page = recordReader.getNextPage();
if (page == null) {
close();
return null;
}
return page;
}
catch (PrestoException e) {

48
presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSourceFactory.java
View File

@ -37,16 +37,17 @@ import io.prestosql.plugin.hive.FileFormatDataSourceStats;
import io.prestosql.plugin.hive.HdfsEnvironment;
import io.prestosql.plugin.hive.HiveColumnHandle;
import io.prestosql.plugin.hive.HiveConfig;
import io.prestosql.plugin.hive.HivePageSourceProvider;
import io.prestosql.plugin.hive.HiveSelectivePageSourceFactory;
import io.prestosql.plugin.hive.HiveSessionProperties;
import io.prestosql.plugin.hive.HiveType;
import io.prestosql.plugin.hive.HiveUtil;
import io.prestosql.plugin.hive.coercions.HiveCoercer;
import io.prestosql.plugin.hive.orc.OrcPageSource.ColumnAdaptation;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ConnectorPageSource;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.connector.FixedPageSource;
import io.prestosql.spi.dynamicfilter.DynamicFilter;
import io.prestosql.spi.heuristicindex.IndexMetadata;
import io.prestosql.spi.predicate.Domain;
import io.prestosql.spi.predicate.TupleDomain;
@ -75,6 +76,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -157,11 +160,12 @@ public class OrcSelectivePageSourceFactory
TupleDomain<HiveColumnHandle> domainPredicate,
Optional<List<TupleDomain<HiveColumnHandle>>> additionPredicates,
DateTimeZone hiveStorageTimeZone,
Map<ColumnHandle, DynamicFilter> dynamicFilter,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
Optional<List<IndexMetadata>> indexes,
boolean splitCacheable)
boolean splitCacheable,
List<HivePageSourceProvider.ColumnMapping> columnMappings,
Map<Integer, HiveCoercer> coercers)
{
if (!HiveUtil.isDeserializerClass(schema, OrcSerde.class)) {
return Optional.empty();
@ -185,7 +189,7 @@ public class OrcSelectivePageSourceFactory
return Optional.of(createOrcPageSource(
hdfsEnvironment,
session.getUser(),
session,
configuration,
path,
start,
@ -207,14 +211,15 @@ public class OrcSelectivePageSourceFactory
getOrcLazyReadSmallRanges(session),
isOrcBloomFiltersEnabled(session),
stats,
dynamicFilter,
deleteDeltaLocations,
startRowOffsetOfFile,
indexes,
orcCacheStore,
orcCacheProperties,
additionPredicates.orElseGet(() -> ImmutableList.of()),
positions));
positions,
columnMappings,
coercers));
/* Todo(Nitin): For Append Pattern
appendPredicates.get().stream().forEach(newDomainPredicate ->
@ -242,7 +247,6 @@ public class OrcSelectivePageSourceFactory
getOrcLazyReadSmallRanges(session),
isOrcBloomFiltersEnabled(session),
stats,
dynamicFilter,
deleteDeltaLocations,
startRowOffsetOfFile,
indexes,
@ -258,7 +262,7 @@ public class OrcSelectivePageSourceFactory
return Optional.of(createOrcPageSource(
hdfsEnvironment,
session.getUser(),
session,
configuration,
path,
start,
@ -280,19 +284,20 @@ public class OrcSelectivePageSourceFactory
getOrcLazyReadSmallRanges(session),
isOrcBloomFiltersEnabled(session),
stats,
dynamicFilter,
deleteDeltaLocations,
startRowOffsetOfFile,
indexes,
orcCacheStore,
orcCacheProperties,
ImmutableList.of(),
null));
null,
columnMappings,
coercers));
}
public static OrcSelectivePageSource createOrcPageSource(
HdfsEnvironment hdfsEnvironment,
String sessionUser,
ConnectorSession session,
Configuration configuration,
Path path,
long start,
@ -314,17 +319,18 @@ public class OrcSelectivePageSourceFactory
boolean lazyReadSmallRanges,
boolean orcBloomFiltersEnabled,
FileFormatDataSourceStats stats,
Map<ColumnHandle, DynamicFilter> dynamicFilter,
Optional<DeleteDeltaLocations> deleteDeltaLocations,
Optional<Long> startRowOffsetOfFile,
Optional<List<IndexMetadata>> indexes,
OrcCacheStore orcCacheStore,
OrcCacheProperties orcCacheProperties,
List<TupleDomain<HiveColumnHandle>> additionalDomainPredicates,
List<Integer> positions)
List<Integer> positions,
List<HivePageSourceProvider.ColumnMapping> columnMappings,
Map<Integer, HiveCoercer> coercers)
{
checkArgument(!domainPredicate.isNone(), "Unexpected NONE domain");
String sessionUser = session.getUser();
OrcDataSource orcDataSource;
try {
//Always create a lazy Stream. HDFS stream opened only when required.
@ -401,6 +407,7 @@ public class OrcSelectivePageSourceFactory
additionalDomainPredicates.stream()
.forEach(ap -> ap.getDomains().get().forEach((k, v) -> additionalPredicateDomains.merge(k, v, (v1, v2) -> v1.union(v2))));
Map<String, List<Domain>> orDomains = new ConcurrentHashMap<>();
for (HiveColumnHandle column : columns) {
OrcColumn orcColumn = null;
if (useOrcColumnNames || isFullAcid) {
@ -425,6 +432,7 @@ public class OrcSelectivePageSourceFactory
domain = additionalPredicateDomains.get(column);
if (domain != null) {
predicateBuilder.addOrColumn(orcColumn.getColumnId(), domain);
orDomains.computeIfAbsent(column.getName(), l -> new ArrayList<>()).add(domain);
}
}
else if (isFullAcid && readType instanceof RowType && column.getName().equalsIgnoreCase("row__id")) {
@ -486,7 +494,10 @@ public class OrcSelectivePageSourceFactory
orcCacheProperties,
Optional.empty(),
orFilters,
positions);
positions,
HiveSessionProperties.isOrcPushdownDataCacheEnabled(session),
Maps.transformValues(coercers, Function.class::cast),
orDomains);
OrcDeletedRows deletedRows = new OrcDeletedRows(
path.getName(),
@ -508,7 +519,10 @@ public class OrcSelectivePageSourceFactory
// deletedRows,
// isFullAcid && indexes.isPresent(),
systemMemoryUsage,
stats);
stats,
session,
columnMappings,
typeManager);
}
catch (Exception e) {
try {

2
presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java
View File

@ -92,7 +92,7 @@ public class IndexCache
.map(HiveColumnHandle::getName)
.map(String::toLowerCase).forEach(column -> {
String indexCacheKeyPath = Paths.get(tableFqn, column, pathUri.getPath()).toString();
IndexCacheKey indexCacheKey = new IndexCacheKey(indexCacheKeyPath, lastModifiedTime, "bitmap");
IndexCacheKey indexCacheKey = new IndexCacheKey(indexCacheKeyPath, lastModifiedTime, "bitmap", "bloom");
// check if cache contains the key
List<IndexMetadata> predicateIndexes = cache.getIfPresent(indexCacheKey);

3
presto-main/src/main/java/io/prestosql/query/CachedSqlQueryExecution.java
View File

@ -372,7 +372,8 @@ public class CachedSqlQueryExecution
connectorTransactionHandleMap.put(tableHandle.getTransaction(), newTableHandle.getTransaction());
// Return a new table handle with the ID, output symbols, assignments, and enforced constraints of the cached table handle
return new TableScanNode(node.getId(), newTableHandle, node.getOutputSymbols(), node.getAssignments(), node.getEnforcedConstraint());
return new TableScanNode(node.getId(), newTableHandle, node.getOutputSymbols(), node.getAssignments(),
node.getEnforcedConstraint(), node.getPredicate());
}
@Override

3
presto-main/src/main/java/io/prestosql/sql/builder/optimizer/SubQueryPushDown.java
View File

@ -316,7 +316,8 @@ public class SubQueryPushDown
result.get().getHandle(),
symbolsBuilder.build(),
columnHandleBuilder.build(),
TupleDomain.all()),
TupleDomain.all(),
Optional.empty()),
new Assignments(assignmentsBuilder.build()));
return Optional.of(output);

3
presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java
View File

@ -795,7 +795,8 @@ public class PlanFragmenter
newTable,
node.getOutputSymbols(),
node.getAssignments(),
node.getEnforcedConstraint());
node.getEnforcedConstraint(),
node.getPredicate());
}
}
}

32
presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java
View File

@ -422,7 +422,7 @@ public class PlanOptimizers
ImmutableSet.<Rule<?>>builder()
.addAll(projectionPushdownRules)
.add(new PushLimitIntoTableScan(metadata))
.add(new PushPredicateIntoTableScan(metadata, typeAnalyzer))
//.add(new PushPredicateIntoTableScan(metadata, typeAnalyzer))
.add(new PushSampleIntoTableScan(metadata))
.build()),
new IterativeOptimizer(
@ -477,11 +477,11 @@ public class PlanOptimizers
ImmutableSet.of(new EliminateCrossJoins())), // This can pull up Filter and Project nodes from between Joins, so we need to push them down again
predicatePushDown,
simplifyOptimizer, // Should be always run after PredicatePushDown
new IterativeOptimizer(
/*new IterativeOptimizer(
ruleStats,
statsCalculator,
estimatedExchangesCostCalculator,
ImmutableSet.of(new PushPredicateIntoTableScan(metadata, typeAnalyzer))),
ImmutableSet.of(new PushPredicateIntoTableScan(metadata, typeAnalyzer))),*/
projectionPushDown,
new PruneUnreferencedOutputs(),
new IterativeOptimizer(
@ -524,12 +524,6 @@ public class PlanOptimizers
.add(new InlineProjections())
.build()));
builder.add(new IterativeOptimizer(
ruleStats,
statsCalculator,
costCalculator,
ImmutableSet.of(new PushDeleteIntoConnector(metadata)))); // Must run before AddExchanges
if (!forceSingleNode) {
builder.add(new ReplicateSemiJoinInDelete()); // Must run before AddExchanges
builder.add((new IterativeOptimizer(
@ -549,6 +543,23 @@ public class PlanOptimizers
ImmutableSet.of(new PushTableWriteThroughUnion()))); // Must run before AddExchanges
builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchanges(metadata, typeAnalyzer)));
}
IterativeOptimizer pushdownRule = new IterativeOptimizer(
ruleStats,
statsCalculator,
estimatedExchangesCostCalculator,
ImmutableSet.of(new PushPredicateIntoTableScan(metadata, typeAnalyzer)));
IterativeOptimizer pushdownDeleteRule = new IterativeOptimizer(
ruleStats,
statsCalculator,
costCalculator,
ImmutableSet.of(new PushDeleteIntoConnector(metadata, false))); // Must run before AddExchanges
IterativeOptimizer pushdownDeleteWithExchangeRule = new IterativeOptimizer(
ruleStats,
statsCalculator,
costCalculator,
ImmutableSet.of(new PushDeleteIntoConnector(metadata, true))); // Must run before AddExchanges
//noinspection UnusedAssignment
estimatedExchangesCostCalculator = null; // Prevent accidental use after AddExchanges
@ -576,6 +587,9 @@ public class PlanOptimizers
.add(new PushRemoteExchangeThroughAssignUniqueId())
.add(new InlineProjections())
.build()));
builder.add(pushdownRule);
builder.add(pushdownDeleteWithExchangeRule);
builder.add(pushdownDeleteRule);
// Optimizers above this don't understand local exchanges, so be careful moving this.
builder.add(new AddLocalExchanges(metadata, typeAnalyzer));

3
presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PruneTableScanColumns.java
View File

@ -42,6 +42,7 @@ public class PruneTableScanColumns
tableScanNode.getTable(),
filteredCopy(tableScanNode.getOutputSymbols(), referencedOutputs::contains),
filterKeys(tableScanNode.getAssignments(), referencedOutputs::contains),
tableScanNode.getEnforcedConstraint()));
tableScanNode.getEnforcedConstraint(),
tableScanNode.getPredicate()));
}
}

21
presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushDeleteIntoConnector.java
View File

@ -25,6 +25,7 @@ import io.prestosql.sql.planner.plan.TableScanNode;
import static com.google.common.collect.Iterables.getOnlyElement;
import static io.prestosql.matching.Capture.newCapture;
import static io.prestosql.sql.planner.plan.Patterns.delete;
import static io.prestosql.sql.planner.plan.Patterns.exchange;
import static io.prestosql.sql.planner.plan.Patterns.source;
import static io.prestosql.sql.planner.plan.Patterns.tableFinish;
import static io.prestosql.sql.planner.plan.Patterns.tableScan;
@ -34,22 +35,34 @@ public class PushDeleteIntoConnector
implements Rule<TableFinishNode>
{
private static final Capture<TableScanNode> TABLE_SCAN = newCapture();
private static final Pattern<TableFinishNode> PATTERN =
private static final Pattern<TableFinishNode> PATTERN_WITHOUT_EXCAHNGE =
tableFinish().with(source().matching(
delete().with(source().matching(
tableScan().capturedAs(TABLE_SCAN)))));
private final Metadata metadata;
private static final Pattern<TableFinishNode> PATTERN_WITH_EXCAHNGE =
tableFinish().with(source().matching(
exchange().with(source().matching(
delete().with(source().matching(
tableScan().capturedAs(TABLE_SCAN)))))));
public PushDeleteIntoConnector(Metadata metadata)
private final Metadata metadata;
private final boolean withExchange;
public PushDeleteIntoConnector(Metadata metadata, boolean withExchange)
{
this.metadata = requireNonNull(metadata, "metadata is null");
this.withExchange = withExchange;
}
@Override
public Pattern<TableFinishNode> getPattern()
{
return PATTERN;
if (withExchange) {
return PATTERN_WITH_EXCAHNGE;
}
return PATTERN_WITHOUT_EXCAHNGE;
}
@Override

3
presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushLimitIntoTableScan.java
View File

@ -61,7 +61,8 @@ public class PushLimitIntoTableScan
result.getHandle(),
tableScan.getOutputSymbols(),
tableScan.getAssignments(),
tableScan.getEnforcedConstraint());
tableScan.getEnforcedConstraint(),
tableScan.getPredicate());
if (!result.isLimitGuaranteed()) {
node = new LimitNode(limit.getId(), node, limit.getCount(), limit.isPartial());

13
presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushPredicateIntoTableScan.java
View File

@ -57,7 +57,6 @@ import java.util.stream.Collectors;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.collect.Sets.intersection;
import static io.prestosql.SystemSessionProperties.isPushTableWriteThroughUnion;
import static io.prestosql.matching.Capture.newCapture;
import static io.prestosql.metadata.TableLayoutResult.computeEnforced;
import static io.prestosql.sql.ExpressionUtils.combineConjuncts;
@ -170,7 +169,7 @@ public class PushPredicateIntoTableScan
AtomicInteger i = new AtomicInteger(0);
additionalPredicates.stream().forEach(er -> {
log.warn("[%d]- Enforced [%s]\n\tRemaining [%s]", i.getAndIncrement(),
log.debug("[%d]- Enforced [%s]\n\tRemaining [%s]", i.getAndIncrement(),
er.getTupleDomain(), er.getRemainingExpression());
});
@ -184,10 +183,10 @@ public class PushPredicateIntoTableScan
Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
Boolean isPushDownEnabled = isPushTableWriteThroughUnion(session);
Constraint constraint;
List<Constraint> additionalConstraints = orDomains.stream().map(d -> new Constraint(d))
List<Constraint> additionalConstraints = orDomains.stream()
.filter(d -> !d.isAll() && !d.isNone())
.map(d -> new Constraint(d))
.collect(Collectors.toList());
if (pruneWithPredicateExpression) {
@ -220,7 +219,6 @@ public class PushPredicateIntoTableScan
return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
}
constraint.setPushDownEnabled(isPushDownEnabled);
Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, node.getTable(), constraint, additionalConstraints);
if (!result.isPresent()) {
@ -257,7 +255,8 @@ public class PushPredicateIntoTableScan
newTable,
node.getOutputSymbols(),
node.getAssignments(),
computeEnforced(newDomain, remainingFilter));
computeEnforced(newDomain, remainingFilter),
Optional.of(deterministicPredicate));
// The order of the arguments to combineConjuncts matters:
// * Unenforced constraints go first because they can only be simple column references,

3
presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushSampleIntoTableScan.java
View File

@ -61,7 +61,8 @@ public class PushSampleIntoTableScan
result,
tableScan.getOutputSymbols(),
tableScan.getAssignments(),
tableScan.getEnforcedConstraint())))
tableScan.getEnforcedConstraint(),
tableScan.getPredicate())))
.orElseGet(Result::empty);
}

3
presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java
View File

@ -236,7 +236,8 @@ public class BeginTableWrite
handle,
scan.getOutputSymbols(),
scan.getAssignments(),
scan.getEnforcedConstraint());
scan.getEnforcedConstraint(),
scan.getPredicate());
}
if (node instanceof FilterNode) {

3
presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneUnreferencedOutputs.java
View File

@ -442,7 +442,8 @@ public class PruneUnreferencedOutputs
node.getTable(),
newOutputs,
newAssignments,
node.getEnforcedConstraint());
node.getEnforcedConstraint(),
node.getPredicate());
}
@Override

19
presto-main/src/main/java/io/prestosql/sql/planner/plan/TableScanNode.java
View File

@ -21,11 +21,13 @@ import io.prestosql.metadata.TableHandle;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.sql.planner.Symbol;
import io.prestosql.sql.tree.Expression;
import javax.annotation.concurrent.Immutable;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.base.Preconditions.checkArgument;
@ -41,6 +43,7 @@ public class TableScanNode
private final Map<Symbol, ColumnHandle> assignments; // symbol -> column
private final TupleDomain<ColumnHandle> enforcedConstraint;
private final Optional<Expression> predicate;
// We need this factory method to disambiguate with the constructor used for deserializing
// from a json object. The deserializer sets some fields which are never transported
@ -51,7 +54,7 @@ public class TableScanNode
List<Symbol> outputs,
Map<Symbol, ColumnHandle> assignments)
{
return new TableScanNode(id, table, outputs, assignments, TupleDomain.all());
return new TableScanNode(id, table, outputs, assignments, TupleDomain.all(), Optional.empty());
}
@JsonCreator
@ -59,7 +62,8 @@ public class TableScanNode
@JsonProperty("id") PlanNodeId id,
@JsonProperty("table") TableHandle table,
@JsonProperty("outputSymbols") List<Symbol> outputs,
@JsonProperty("assignments") Map<Symbol, ColumnHandle> assignments)
@JsonProperty("assignments") Map<Symbol, ColumnHandle> assignments,
@JsonProperty("predicate") Optional<Expression> predicate)
{
// This constructor is for JSON deserialization only. Do not use.
super(id);
@ -68,6 +72,7 @@ public class TableScanNode
this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null"));
checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs");
this.enforcedConstraint = null;
this.predicate = predicate;
}
public TableScanNode(
@ -75,7 +80,8 @@ public class TableScanNode
TableHandle table,
List<Symbol> outputs,
Map<Symbol, ColumnHandle> assignments,
TupleDomain<ColumnHandle> enforcedConstraint)
TupleDomain<ColumnHandle> enforcedConstraint,
Optional<Expression> predicate)
{
super(id);
this.table = requireNonNull(table, "table is null");
@ -83,6 +89,7 @@ public class TableScanNode
this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null"));
checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs");
this.enforcedConstraint = requireNonNull(enforcedConstraint, "enforcedConstraint is null");
this.predicate = requireNonNull(predicate, "predicate expression cannot be empty");
}
@JsonProperty("table")
@ -119,6 +126,12 @@ public class TableScanNode
return enforcedConstraint;
}
@JsonProperty("predicate")
public Optional<Expression> getPredicate()
{
return predicate;
}
@Override
public List<PlanNode> getSources()
{

38
presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java
View File

@ -759,6 +759,12 @@ public class PlanPrinter
formatString += "grouped = %s, ";
arguments.add(stageExecutionStrategy.get().isScanGroupedExecution(scanNode.get().getId()));
}
String pushdownPredicates = printTablePushDownFilters(scanNode.get());
if (pushdownPredicates.length() > 0) {
formatString += "pushdownFilters = %s, ";
arguments.add(pushdownPredicates);
}
}
if (filterNode.isPresent()) {
@ -830,6 +836,38 @@ public class PlanPrinter
.collect(Collectors.joining(", ", "{", "}"));
}
private String printTablePushDownFilters(TableScanNode node)
{
StringBuilder str = new StringBuilder();
TupleDomain<ColumnHandle> predicate = tableInfoSupplier.apply(node).getPredicate();
if (!predicate.isNone() && !predicate.isAll()) {
if (predicate.isAll() && (!node.getEnforcedConstraint().isAll() || !node.getEnforcedConstraint().isNone())) {
predicate = node.getEnforcedConstraint();
}
if (!predicate.isNone() && !predicate.isAll()) {
str.append("[ ");
str.append(predicate.getDomains().get()
.entrySet().stream()
.map(filter -> filter.getKey() + " <- " + formatDomain(filter.getValue().simplify()))
.collect(Collectors.joining(" AND ", "{", "}")));
str.append(" ]");
if (node.getTable().getConnectorHandle().hasAdditionalFiltersPushdown()) {
str.append("[ ");
str.append(" AND ");
str.append(" ]");
}
}
if (node.getTable().getConnectorHandle().hasAdditionalFiltersPushdown()) {
str.append(node.getTable().getConnectorHandle()
.getAdditionalFilterConditions((domain) -> formatDomain(domain)));
}
}
return str.toString();
}
private void printTableScanInfo(NodeRepresentation nodeOutput, TableScanNode node)
{
TupleDomain<ColumnHandle> predicate = tableInfoSupplier.apply(node).getPredicate();

119
presto-main/src/main/java/io/prestosql/utils/PredicateExtractor.java
View File

@ -44,6 +44,7 @@ import java.sql.Timestamp;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Queue;
@ -68,52 +69,87 @@ public class PredicateExtractor
{
}
private static Optional<FilterNode> getFilterNode(SqlStageExecution stage)
private static List<PlanNode> getFilterNode(SqlStageExecution stage)
{
PlanFragment fragment = stage.getFragment();
PlanNode root = fragment.getRoot();
List<PlanNode> result = new LinkedList<>();
Queue<PlanNode> queue = new LinkedList<>();
queue.add(root);
while (!queue.isEmpty()) {
PlanNode node = queue.poll();
if (node instanceof FilterNode) {
return Optional.of((FilterNode) node);
if (node instanceof FilterNode
|| node instanceof TableScanNode) {
result.add(node);
}
queue.addAll(node.getSources());
}
return Optional.empty();
return result;
}
public static boolean isSplitFilterApplicable(SqlStageExecution stage)
{
Optional<FilterNode> filterNodeOptional = getFilterNode(stage);
List<PlanNode> filterNodeOptional = getFilterNode(stage);
if (!filterNodeOptional.isPresent()) {
if (filterNodeOptional.isEmpty()) {
return false;
}
FilterNode filterNode = filterNodeOptional.get();
PlanNode node = filterNodeOptional.get(0);
PlanNode sourceNode = filterNode.getSource();
if (!(sourceNode instanceof TableScanNode)) {
return false;
if (node instanceof FilterNode) {
FilterNode filterNode = (FilterNode) node;
PlanNode sourceNode = filterNode.getSource();
if (!(sourceNode instanceof TableScanNode)) {
return false;
}
//if a catalog name starts with a $, it's not an normal query, could be something like show tables;
TableHandle table = ((TableScanNode) sourceNode).getTable();
String catalogName = table.getCatalogName().getCatalogName();
if (catalogName.startsWith("$")) {
return false;
}
/* (!(table.getConnectorHandle().isFilterSupported()
* && (isSupportedExpression(filterNode.getPredicate())
* || (((TableScanNode) sourceNode).getPredicate().isPresent()
* && isSupportedExpression(((TableScanNode) sourceNode).getPredicate().get())))))
*/
if (!table.getConnectorHandle().isFilterSupported()) {
return false;
}
if (!isSupportedExpression(filterNode.getPredicate())
&& (!((TableScanNode) sourceNode).getPredicate().isPresent()
|| !isSupportedExpression(((TableScanNode) sourceNode).getPredicate().get()))) {
return false;
}
}
//if a catalog name starts with a $, it's not an normal query, could be something like show tables;
TableHandle table = ((TableScanNode) sourceNode).getTable();
String catalogName = table.getCatalogName().getCatalogName();
if (catalogName.startsWith("$")) {
return false;
if (node instanceof TableScanNode) {
TableScanNode tableScanNode = (TableScanNode) node;
//if a catalog name starts with a $, it's not an normal query, could be something like show tables;
TableHandle table = tableScanNode.getTable();
String catalogName = table.getCatalogName().getCatalogName();
if (catalogName.startsWith("$")) {
return false;
}
if (!table.getConnectorHandle().isFilterSupported()) {
return false;
}
if (!tableScanNode.getPredicate().isPresent()
|| !isSupportedExpression(tableScanNode.getPredicate().get())) {
return false;
}
}
boolean supported = table.getConnectorHandle().isFilterSupported();
if (!supported || !isSupportedExpression(filterNode.getPredicate())) {
return false;
}
return true;
}
@ -142,6 +178,7 @@ public class PredicateExtractor
if (predicate instanceof InPredicate) {
return true;
}
if (predicate instanceof NotExpression) {
return true;
}
@ -158,31 +195,53 @@ public class PredicateExtractor
*/
public static Tuple<Optional<Expression>, Map<Symbol, ColumnHandle>> getExpression(SqlStageExecution stage)
{
Optional<FilterNode> filterNodeOptional = getFilterNode(stage);
List<PlanNode> filterNodeOptional = getFilterNode(stage);
if (!filterNodeOptional.isPresent()) {
if (filterNodeOptional.size() == 0) {
return new Tuple<>(Optional.empty(), new HashMap<>());
}
FilterNode filterNode = filterNodeOptional.get();
if (filterNode.getSource() instanceof TableScanNode) {
TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
Expression predicate = filterNode.getPredicate();
return new Tuple<>(Optional.of(predicate), tableScanNode.getAssignments());
if (filterNodeOptional.get(0) instanceof FilterNode) {
FilterNode filterNode = (FilterNode) filterNodeOptional.get(0);
if (filterNode.getSource() instanceof TableScanNode) {
TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
if (tableScanNode.getPredicate().isPresent()) {
return new Tuple<>(tableScanNode.getPredicate(), tableScanNode.getAssignments());
}
return new Tuple<>(Optional.of(filterNode.getPredicate()), tableScanNode.getAssignments());
}
return new Tuple<>(Optional.empty(), new HashMap<>());
}
if (filterNodeOptional.get(0) instanceof TableScanNode) {
TableScanNode tableScanNode = (TableScanNode) filterNodeOptional.get(0);
if (tableScanNode.getPredicate().isPresent()) {
return new Tuple<>(tableScanNode.getPredicate(), tableScanNode.getAssignments());
}
}
return new Tuple<>(Optional.empty(), new HashMap<>());
}
public static Optional<String> getFullyQualifiedName(SqlStageExecution stage)
{
Optional<FilterNode> filterNodeOptional = getFilterNode(stage);
List<PlanNode> filterNodeOptional = getFilterNode(stage);
if (!filterNodeOptional.isPresent()) {
if (filterNodeOptional.size() == 0) {
return Optional.empty();
}
FilterNode filterNode = filterNodeOptional.get();
TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
TableScanNode tableScanNode;
if (filterNodeOptional.get(0) instanceof FilterNode) {
FilterNode filterNode = (FilterNode) filterNodeOptional.get(0);
tableScanNode = (TableScanNode) filterNode.getSource();
}
else {
tableScanNode = (TableScanNode) filterNodeOptional.get(0);
}
String fullQualifiedTableName = tableScanNode.getTable().getFullyQualifiedName();
return Optional.of(fullQualifiedTableName);

1
presto-main/src/main/java/io/prestosql/vacuum/AutoVacuumScanner.java
View File

@ -271,7 +271,6 @@ public class AutoVacuumScanner
return connectorMetadata.getTablesForVacuum();
}
catch (UnsupportedOperationException e) {
log.error("Catalog[%s] does not support auto vacuum scan", catalog.getCatalogName());
return null;
}
catch (Exception e) {

3
presto-main/src/test/java/io/prestosql/cost/TestCostCalculator.java
View File

@ -797,7 +797,8 @@ public class TestCostCalculator
new TableHandle(new CatalogName("tpch"), tableHandle, INSTANCE, Optional.of(new TpchTableLayoutHandle(tableHandle, TupleDomain.all()))),
symbolsList,
assignments.build(),
TupleDomain.all());
TupleDomain.all(),
Optional.empty());
}
private PlanNode project(String id, PlanNode source, String symbol, Expression expression)

21
presto-main/src/test/java/io/prestosql/sql/planner/TestEffectivePredicateExtractor.java
View File

@ -409,7 +409,8 @@ public class TestEffectivePredicateExtractor
makeTableHandle(TupleDomain.none()),
ImmutableList.copyOf(assignments.keySet()),
assignments,
TupleDomain.none());
TupleDomain.none(),
Optional.empty());
effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(effectivePredicate, FALSE_LITERAL);
TupleDomain<ColumnHandle> predicate = TupleDomain.withColumnDomains(ImmutableMap.of(scanAssignments.get(A), Domain.singleValue(BIGINT, 1L)));
@ -418,7 +419,8 @@ public class TestEffectivePredicateExtractor
makeTableHandle(predicate),
ImmutableList.copyOf(assignments.keySet()),
assignments,
predicate);
predicate,
Optional.empty());
effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(1L), AE)));
predicate = TupleDomain.withColumnDomains(ImmutableMap.of(
@ -429,7 +431,8 @@ public class TestEffectivePredicateExtractor
makeTableHandle(TupleDomain.withColumnDomains(ImmutableMap.of(scanAssignments.get(A), Domain.singleValue(BIGINT, 1L)))),
ImmutableList.copyOf(assignments.keySet()),
assignments,
predicate);
predicate,
Optional.empty());
effectivePredicate = effectivePredicateExtractorWithoutTableProperties.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(2L), BE), equals(bigintLiteral(1L), AE)));
@ -438,7 +441,8 @@ public class TestEffectivePredicateExtractor
makeTableHandle(predicate),
ImmutableList.copyOf(assignments.keySet()),
assignments,
TupleDomain.all());
TupleDomain.all(),
Optional.empty());
effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(effectivePredicate, and(equals(AE, bigintLiteral(1)), equals(BE, bigintLiteral(2))));
@ -449,7 +453,8 @@ public class TestEffectivePredicateExtractor
assignments,
TupleDomain.withColumnDomains(ImmutableMap.of(
scanAssignments.get(A), Domain.multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L)),
scanAssignments.get(B), Domain.multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L)))));
scanAssignments.get(B), Domain.multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L)))),
Optional.empty());
effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(2L), BE), equals(bigintLiteral(1L), AE)));
@ -458,7 +463,8 @@ public class TestEffectivePredicateExtractor
makeTableHandle(TupleDomain.all()),
ImmutableList.copyOf(assignments.keySet()),
assignments,
TupleDomain.all());
TupleDomain.all(),
Optional.empty());
effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer);
assertEquals(effectivePredicate, BooleanLiteral.TRUE_LITERAL);
}
@ -929,7 +935,8 @@ public class TestEffectivePredicateExtractor
makeTableHandle(TupleDomain.all()),
ImmutableList.copyOf(scanAssignments.keySet()),
scanAssignments,
TupleDomain.all());
TupleDomain.all(),
Optional.empty());
}
private static PlanNodeId newId()

5
presto-main/src/test/java/io/prestosql/sql/planner/TestLogicalPlanner.java
View File

@ -736,10 +736,11 @@ public class TestLogicalPlanner
assertPlan(
"SELECT orderkey FROM orders WHERE orderstatus='F'",
output(
constrainedTableScanWithTableLayout(
project(
constrainedTableScanWithTableLayout(
"orders",
ImmutableMap.of("orderstatus", singleValue(createVarcharType(1), utf8Slice("F"))),
ImmutableMap.of("orderkey", "orderkey"))));
ImmutableMap.of("orderkey", "orderkey")))));
}
@Test

2
presto-main/src/test/java/io/prestosql/sql/planner/TestPredicatePushdown.java
View File

@ -477,7 +477,7 @@ public class TestPredicatePushdown
"orders",
ImmutableMap.of("ORDERSTATUS", "orderstatus"))),
anyTree(
filter("CAST(NAME AS varchar(1)) IN ('F', 'O')",
filter("CAST(NAME AS varchar(1)) BETWEEN 'A' AND 'O'",
tableScan(
"nation",
ImmutableMap.of("NAME", "name")))))));

3
presto-main/src/test/java/io/prestosql/sql/planner/TestTypeValidator.java
View File

@ -97,7 +97,8 @@ public class TestTypeValidator
TEST_TABLE_HANDLE,
ImmutableList.copyOf(assignments.keySet()),
assignments,
TupleDomain.all());
TupleDomain.all(),
Optional.empty());
}
@Test

3
presto-main/src/test/java/io/prestosql/sql/planner/iterative/rule/test/PlanBuilder.java
View File

@ -456,7 +456,8 @@ public class PlanBuilder
tableHandle,
symbols,
assignments,
enforcedConstraint);
enforcedConstraint,
Optional.empty());
}
public TableFinishNode tableDelete(SchemaTableName schemaTableName, PlanNode deleteSource, Symbol deleteRowId)

3
presto-main/src/test/java/io/prestosql/utils/TestUtil.java
View File

@ -120,7 +120,8 @@ public class TestUtil
makeTableHandle(TupleDomain.none()),
ImmutableList.copyOf(assignments.keySet()),
assignments,
TupleDomain.none());
TupleDomain.none(),
Optional.empty());
PlanBuilder planBuilder = new PlanBuilder(new PlanNodeIdAllocator(), dummyMetadata());
FilterNode filterNode = planBuilder.filter(expr, tableScanNode);

113
presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java
View File

@ -36,7 +36,8 @@ import io.prestosql.orc.metadata.statistics.ColumnStatistics;
import io.prestosql.orc.metadata.statistics.StripeStatistics;
import io.prestosql.orc.reader.AbstractColumnReader;
import io.prestosql.orc.reader.CachingColumnReader;
import io.prestosql.orc.reader.SelectiveCachingColumnReader;
import io.prestosql.orc.reader.DataCachingSelectiveColumnReader;
import io.prestosql.orc.reader.ResultCachingSelectiveColumnReader;
import io.prestosql.orc.stream.InputStreamSources;
import io.prestosql.orc.stream.StreamSourceMeta;
import io.prestosql.spi.Page;
@ -153,7 +154,8 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
Optional<List<IndexMetadata>> indexes,
Map<String, Domain> domains,
OrcCacheStore orcCacheStore,
OrcCacheProperties orcCacheProperties)
OrcCacheProperties orcCacheProperties,
Map<String, List<Domain>> orDomains)
throws OrcCorruptionException
{
requireNonNull(readColumns, "readColumns is null");
@ -183,6 +185,7 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
// it is possible that old versions of orc use 0 to mean there are no row groups
checkArgument(rowsInRowGroup > 0, "rowsInRowGroup must be greater than zero");
checkArgument(orDomains != null, "orDomain map cannot be null");
// sort stripes by file position
List<StripeInfo> stripeInfos = new ArrayList<>();
@ -225,14 +228,12 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
for (int i = 0; i < stripeInfos.size(); i++) {
StripeInfo info = stripeInfos.get(i);
StripeInformation stripe = info.getStripe();
if (splitContainsStripe(splitOffset, splitLength, stripe) && isStripeIncluded(stripe, info.getStats(), predicate)) {
if (splitContainsStripe(splitOffset, splitLength, stripe)
&& isStripeIncluded(stripe, info.getStats(), predicate)
&& !filterStripeUsingIndex(stripe, stripeOffsetToIndex, domains, orDomains)) {
stripes.add(stripe);
stripeFilePositions.add(fileRowCount);
totalRowCount += stripe.getNumberOfRows();
if (!stripeOffsetToIndex.isEmpty()) {
stripeIndexes.put(stripe, stripeOffsetToIndex.get(stripe.getOffset()));
}
}
fileRowCount += stripe.getNumberOfRows();
}
@ -241,37 +242,6 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
this.stripes = stripes.build();
this.stripeFilePositions = stripeFilePositions.build();
// now that we know which stripes will be read, apply indexes on them if applicable
// i.e. if an index exists for the pushed down predicates
// once the indexes are applied, for each stripe we will have the rows inside
// the stripe that matched the predicates
stripeIndexes.entrySet().stream().forEach(stripeIndex -> {
Map<Index, Domain> indexDomainMap = new HashMap<>();
for (Map.Entry<String, Domain> domainEntry : domains.entrySet()) {
String columnName = domainEntry.getKey();
Domain columnDomain = domainEntry.getValue();
// if the index exists, there should only be one index for this column within this stripe
List<IndexMetadata> indexMetadata = stripeIndex.getValue().stream().filter(p -> p.getColumn().equalsIgnoreCase(columnName)).collect(Collectors.toList());
if (indexMetadata.isEmpty() || indexMetadata.size() > 1) {
continue;
}
Index index = indexMetadata.get(0).getIndex();
indexDomainMap.put(index, columnDomain);
}
if (!indexDomainMap.isEmpty()) {
Iterator<Integer> thisStripeMatchingRows = indexDomainMap.entrySet().iterator().next().getKey().getMatches(indexDomainMap);
if (thisStripeMatchingRows != null) {
PeekingIterator<Integer> peekingIterator = Iterators.peekingIterator(thisStripeMatchingRows);
stripeMatchingRows.put(stripeIndex.getKey(), peekingIterator);
}
}
});
orcDataSource = wrapWithCacheIfTinyStripes(orcDataSource, this.stripes, maxMergeDistance, tinyStripeThreshold);
this.orcDataSource = orcDataSource;
this.splitLength = splitLength;
@ -314,6 +284,69 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
}
}
private boolean filterStripeUsingIndex(StripeInformation stripe, Map<Long, List<IndexMetadata>> stripeOffsetToIndex,
Map<String, Domain> and, Map<String, List<Domain>> or)
{
if (stripeOffsetToIndex.isEmpty()) {
return false;
}
List<IndexMetadata> stripeIndex = stripeOffsetToIndex.get(Long.valueOf(stripe.getOffset()));
Map<Index, Domain> andDomainMap = new HashMap<>();
Map<Index, Domain> orDomainMap = new HashMap<>();
for (Map.Entry<String, Domain> domainEntry : and.entrySet()) {
String columnName = domainEntry.getKey();
Domain columnDomain = domainEntry.getValue();
// if the index exists, there should only be one index for this column within this stripe
List<IndexMetadata> indexMetadata = stripeIndex.stream().filter(p -> p.getColumn().equalsIgnoreCase(columnName)).collect(Collectors.toList());
if (indexMetadata.isEmpty() || indexMetadata.size() > 1) {
continue;
}
Index index = indexMetadata.get(0).getIndex();
andDomainMap.put(index, columnDomain);
}
for (Map.Entry<String, List<Domain>> domainEntry : or.entrySet()) {
String columnName = domainEntry.getKey();
List<Domain> columnDomain = domainEntry.getValue();
// if the index exists, there should only be one index for this column within this stripe
List<IndexMetadata> indexMetadata = stripeIndex.stream().filter(p -> p.getColumn().equalsIgnoreCase(columnName)).collect(Collectors.toList());
if (indexMetadata.isEmpty() || indexMetadata.size() > 1) {
continue;
}
Index index = indexMetadata.get(0).getIndex();
orDomainMap.put(index, columnDomain.get(0));
}
if (!andDomainMap.isEmpty()) {
Iterator<Integer> thisStripeMatchingRows = ((andDomainMap.entrySet().iterator().next()).getKey()).getMatches(andDomainMap);
if (thisStripeMatchingRows.hasNext()) {
PeekingIterator<Integer> peekingIterator = Iterators.peekingIterator(thisStripeMatchingRows);
if (peekingIterator.peek() != null) {
this.stripeMatchingRows.put(stripe, peekingIterator);
}
return false;
}
return true;
}
if (!orDomainMap.isEmpty()) {
for (Map.Entry<Index, Domain> indexDomainEntry : orDomainMap.entrySet()) {
Iterator<Integer> thisStripeMatchingRows = (indexDomainEntry.getKey()).getMatches(indexDomainEntry.getValue());
if (thisStripeMatchingRows.hasNext()) {
/* any one matched; then include the stripe */
return false;
}
}
return true;
}
return false;
}
private static OptionalInt getFixedWidthRowSize(List<Type> columnTypes)
{
int totalFixedWidth = 0;
@ -642,7 +675,9 @@ abstract class AbstractOrcRecordReader<T extends AbstractColumnReader>
InputStreamSources rowGroupStreamSources = currentRowGroup.getStreamSources();
for (AbstractColumnReader columnReader : columnReaders) {
if (columnReader != null) {
if (columnReader instanceof CachingColumnReader || columnReader instanceof SelectiveCachingColumnReader) {
if (columnReader instanceof CachingColumnReader
|| columnReader instanceof ResultCachingSelectiveColumnReader
|| columnReader instanceof DataCachingSelectiveColumnReader) {
StreamSourceMeta streamSourceMeta = new StreamSourceMeta();
streamSourceMeta.setDataSourceId(orcDataSource.getId());
streamSourceMeta.setStripeOffset(stripes.get(currentStripe).getOffset());

10
presto-orc/src/main/java/io/prestosql/orc/OrcReader.java
View File

@ -28,6 +28,7 @@ import io.prestosql.orc.metadata.OrcType;
import io.prestosql.orc.metadata.OrcType.OrcTypeKind;
import io.prestosql.orc.metadata.PostScript.HiveWriterVersion;
import io.prestosql.spi.Page;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.heuristicindex.IndexMetadata;
import io.prestosql.spi.predicate.Domain;
import io.prestosql.spi.type.Type;
@ -315,7 +316,9 @@ public class OrcReader
OrcCacheProperties orcCacheProperties,
Optional<OrcWriteValidation> writeValidation,
Map<Integer, List<TupleDomainFilter>> additionalFilters,
List<Integer> positions) throws OrcCorruptionException
List<Integer> positions, boolean useDataCache,
Map<Integer, Function<Block, Block>> coercer,
Map<String, List<Domain>> orDomains) throws OrcCorruptionException
{
return new OrcSelectiveRecordReader(
outputColumns,
@ -352,7 +355,10 @@ public class OrcReader
orcCacheStore,
orcCacheProperties,
additionalFilters,
positions);
positions,
useDataCache,
coercer,
orDomains);
}
public static OrcDataSource wrapWithCacheIfTiny(OrcDataSource dataSource, DataSize maxCacheSize)

3
presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java
View File

@ -115,7 +115,8 @@ public class OrcRecordReader
indexes,
domains,
orcCacheStore,
orcCacheProperties);
orcCacheProperties,
ImmutableMap.of());
setColumnReadersParam(createColumnReaders(readColumns, readTypes,
systemMemoryUsage.newAggregatedMemoryContext(),

103
presto-orc/src/main/java/io/prestosql/orc/OrcSelectiveRecordReader.java
View File

@ -13,6 +13,7 @@
*/
package io.prestosql.orc;
import com.google.common.collect.PeekingIterator;
import io.airlift.log.Logger;
import io.airlift.slice.Slice;
import io.airlift.units.DataSize;
@ -25,6 +26,8 @@ import io.prestosql.orc.metadata.PostScript;
import io.prestosql.orc.metadata.StripeInformation;
import io.prestosql.orc.metadata.statistics.ColumnStatistics;
import io.prestosql.orc.metadata.statistics.StripeStatistics;
import io.prestosql.orc.reader.ColumnReader;
import io.prestosql.orc.reader.ColumnReaders;
import io.prestosql.orc.reader.SelectiveColumnReader;
import io.prestosql.orc.reader.SelectiveColumnReaders;
import io.prestosql.spi.Page;
@ -39,14 +42,18 @@ import org.openjdk.jol.info.ClassLayout;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.IntStream;
import static com.google.common.base.Preconditions.checkArgument;
import static io.prestosql.orc.reader.SelectiveColumnReaders.createColumnReader;
import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
public class OrcSelectiveRecordReader
@ -57,17 +64,17 @@ public class OrcSelectiveRecordReader
private final List<Integer> excludePositions;
private final List<Integer> columnReaderOrder;
private final Map<Integer, TupleDomainFilter> filters;
private int[] positions;
List<Integer> outputColumns;
Map<Integer, Type> includedColumns;
private int finalPositionCount;
private int[] finalPositions;
private final Map<Integer, Object> constantValues;
Set<Integer> colReaderWithFilter;
Set<Integer> colReaderWithORFilter;
Set<Integer> colReaderWithoutFilter;
Map<Integer, List<TupleDomainFilter>> additionalFilters;
Map<Integer, Function<Block, Block>> coercers;
public OrcSelectiveRecordReader(
List<Integer> outputColumns,
@ -104,7 +111,10 @@ public class OrcSelectiveRecordReader
OrcCacheStore orcCacheStore,
OrcCacheProperties orcCacheProperties,
Map<Integer, List<TupleDomainFilter>> additionalFilters,
List<Integer> positions)
List<Integer> positions,
boolean useDataCache,
Map<Integer, Function<Block, Block>> coercers,
Map<String, List<Domain>> orDomains)
throws OrcCorruptionException
{
super(fileReadColumns,
@ -134,7 +144,8 @@ public class OrcSelectiveRecordReader
indexes,
domains,
orcCacheStore,
orcCacheProperties);
orcCacheProperties,
orDomains);
int fieldCount = orcTypes.get(OrcColumnId.ROOT_COLUMN).getFieldCount();
this.columnReaderOrder = new ArrayList<>(fieldCount);
@ -143,15 +154,21 @@ public class OrcSelectiveRecordReader
this.includedColumns = includedColumns;
this.excludePositions = positions;
this.filters = filters;
this.additionalFilters = additionalFilters;
this.constantValues = requireNonNull(constantValues, "constantValues is null");
this.coercers = requireNonNull(coercers, "coercers is null");
for (Map.Entry<Integer, Function<Block, Block>> entry : coercers.entrySet()) {
checkArgument(!filters.containsKey(entry.getKey()), "Coercions for columns with range filters are not yet supported");
}
setColumnReadersParam(createColumnReaders(fileColumns,
systemMemoryUsage.newAggregatedMemoryContext(),
new OrcBlockFactory(exceptionTransform, true),
orcCacheStore, orcCacheProperties,
predicate, filters, hiveStorageTimeZone,
outputColumns, includedColumns, orcTypes));
outputColumns, includedColumns, orcTypes, useDataCache));
}
public Page getNextPage()
@ -162,6 +179,7 @@ public class OrcSelectiveRecordReader
return null;
}
matchingRowsInBatchArray = null;
initializePositions(batchSize);
int[] positionsToRead = this.positions;
@ -172,7 +190,7 @@ public class OrcSelectiveRecordReader
if (positionCount != 0) {
for (Integer columnIdx : colReaderWithFilter) {
if (columnReaders[columnIdx] != null) {
positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount);
positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount, filters.get(columnIdx));
if (positionCount == 0) {
break;
}
@ -196,13 +214,13 @@ public class OrcSelectiveRecordReader
int[] newPositions = positionsToRead.clone();
positionCount = updateExcludePositions(positionsToRead, positionCount, accumulator, newPositions);
positionsToRead = newPositions;
positionsToRead = Arrays.copyOf(newPositions, positionCount);
}
if (positionCount != 0) {
for (Integer columnIdx : colReaderWithoutFilter) {
if (columnReaders[columnIdx] != null) {
positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount);
positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount, null);
if (positionCount == 0) {
break;
}
@ -233,6 +251,9 @@ public class OrcSelectiveRecordReader
else {
Block block = getColumnReaders()[columnIndex].getBlock(positionsToRead, positionCount);
updateMaxCombinedBytesPerRow(columnIndex, block);
if (coercers.containsKey(i)) {
block = coercers.get(i).apply(block);
}
blocks[i] = block;
}
}
@ -246,7 +267,38 @@ public class OrcSelectiveRecordReader
private void initializePositions(int batchSize)
{
// currentPosition to currentBatchSize
StripeInformation stripe = stripes.get(currentStripe);
if (matchingRowsInBatchArray == null && stripeMatchingRows.containsKey(stripe)) {
long currentPositionInStripe = currentPosition - currentStripePosition;
PeekingIterator<Integer> matchingRows = stripeMatchingRows.get(stripe);
List<Integer> matchingRowsInBlock = new ArrayList<>();
while (matchingRows.hasNext()) {
Integer row = matchingRows.peek();
if (row >= currentPositionInStripe && row < currentPositionInStripe + batchSize) {
matchingRowsInBlock.add(toIntExact(Long.valueOf(row) - currentPositionInStripe));
matchingRows.next();
}
else if (row >= currentPositionInStripe + currentBatchSize) {
break;
}
}
matchingRowsInBatchArray = new int[matchingRowsInBlock.size()];
IntStream.range(0, matchingRowsInBlock.size()).forEach(
i -> matchingRowsInBatchArray[i] = matchingRowsInBlock.get(i));
log.debug("Find matching rows from stripe. Matching row count for the block = %d", matchingRowsInBatchArray.length);
}
if (positions == null || positions.length < batchSize) {
if (matchingRowsInBatchArray != null) {
positions = matchingRowsInBatchArray;
return;
}
positions = new int[batchSize];
for (int i = 0; i < batchSize; i++) {
positions[i] = i;
@ -284,7 +336,8 @@ public class OrcSelectiveRecordReader
DateTimeZone hiveStorageTimeZone,
List<Integer> outputColumns,
Map<Integer, Type> includedColumns,
ColumnMetadata<OrcType> orcTypes)
ColumnMetadata<OrcType> orcTypes,
boolean useDataCache)
throws OrcCorruptionException
{
int fieldCount = orcTypes.get(OrcColumnId.ROOT_COLUMN).getFieldCount();
@ -298,16 +351,28 @@ public class OrcSelectiveRecordReader
int columnIndex = i;
OrcColumn column = fileColumns.get(columnIndex);
boolean outputRequired = outputColumns.contains(i);
SelectiveColumnReader columnReader = createColumnReader(
orcTypes.get(column.getColumnId()),
column,
Optional.ofNullable(filters.get(i)),
outputRequired ? Optional.of(includedColumns.get(i)) : Optional.empty(),
hiveStorageTimeZone,
systemMemoryContext);
if (orcCacheProperties.isRowDataCacheEnabled()) {
columnReader = SelectiveColumnReaders.wrapWithCachingStreamReader(columnReader, column,
predicate, orcCacheStore.getRowDataCache());
SelectiveColumnReader columnReader = null;
if (useDataCache && orcCacheProperties.isRowDataCacheEnabled()) {
ColumnReader cr = ColumnReaders.createColumnReader(
includedColumns.get(i),
column,
systemMemoryContext,
blockFactory.createNestedBlockFactory(block -> blockLoaded(columnIndex, block)));
columnReader = SelectiveColumnReaders.wrapWithDataCachingStreamReader(cr, column, orcCacheStore.getRowDataCache());
}
else {
columnReader = createColumnReader(
orcTypes.get(column.getColumnId()),
column,
Optional.ofNullable(filters.get(i)),
outputRequired ? Optional.of(includedColumns.get(i)) : Optional.empty(),
hiveStorageTimeZone,
systemMemoryContext);
if (orcCacheProperties.isRowDataCacheEnabled()) {
columnReader = SelectiveColumnReaders.wrapWithResultCachingStreamReader(columnReader, column,
predicate, orcCacheStore.getRowDataCache());
}
}
columnReaders[columnIndex] = columnReader;
if (filters.get(i) != null) {

17
presto-orc/src/main/java/io/prestosql/orc/reader/AbstractDecimalSelectiveColumnReader.java
View File

@ -46,8 +46,8 @@ import static io.prestosql.orc.metadata.Stream.StreamKind.SECONDARY;
import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSource;
import static java.util.Objects.requireNonNull;
public abstract class AbstractDecimalSelectiveColumnReader
implements SelectiveColumnReader
public abstract class AbstractDecimalSelectiveColumnReader<T>
implements SelectiveColumnReader<T>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(AbstractDecimalSelectiveColumnReader.class).instanceSize();
@ -135,11 +135,11 @@ public abstract class AbstractDecimalSelectiveColumnReader
}
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
return readOr(offset, positions, positionCount,
(filter == null) ? null : ImmutableList.of(filter),
(this.filter == null) ? null : ImmutableList.of(this.filter),
null);
}
@ -231,7 +231,7 @@ public abstract class AbstractDecimalSelectiveColumnReader
}
@Override
public Block getBlock(int[] positions, int positionCount)
public Block<T> getBlock(int[] positions, int positionCount)
{
checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
checkState(outputRequired, "This stream reader doesn't produce output");
@ -264,9 +264,8 @@ public abstract class AbstractDecimalSelectiveColumnReader
private void ensureValuesCapacity(int capacity, boolean nullAllowed)
{
int valuesCapacity = valuesPerPosition * capacity;
if (values == null || values.length < valuesCapacity) {
values = new long[valuesCapacity];
if (values == null || values.length < capacity) {
values = new long[valuesPerPosition * capacity];
}
if (nullAllowed) {
@ -278,7 +277,7 @@ public abstract class AbstractDecimalSelectiveColumnReader
abstract void copyValues(int[] positions, int positionsCount, long[] valuesCopy, boolean[] nullsCopy);
abstract Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values);
abstract Block<T> makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values);
abstract void compactValues(int[] positions, int positionCount, boolean compactNulls);

6
presto-orc/src/main/java/io/prestosql/orc/reader/AbstractLongSelectiveColumnReader.java
View File

@ -35,8 +35,8 @@ import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
//TODO: Rajeev: To be combined with LongSelectiveColumnReader
abstract class AbstractLongSelectiveColumnReader
implements SelectiveColumnReader
abstract class AbstractLongSelectiveColumnReader<T>
implements SelectiveColumnReader<T>
{
protected final boolean outputRequired;
@Nullable
@ -63,7 +63,7 @@ abstract class AbstractLongSelectiveColumnReader
}
@Override
public Block mergeBlocks(List<Block> blocks, int positionCount)
public Block<T> mergeBlocks(List<Block<T>> blocks, int positionCount)
{
if (outputType == BIGINT) {
LongArrayBlockBuilder blockBuilder = new LongArrayBlockBuilder(null, positionCount);

9
presto-orc/src/main/java/io/prestosql/orc/reader/BooleanColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.stream.BooleanInputStream;
@ -47,7 +48,7 @@ import static io.prestosql.spi.type.BooleanType.BOOLEAN;
import static java.util.Objects.requireNonNull;
public class BooleanColumnReader
implements ColumnReader<Boolean>
implements ColumnReader<Byte>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(BooleanColumnReader.class).instanceSize();
@ -224,4 +225,10 @@ public class BooleanColumnReader
{
return INSTANCE_SIZE + sizeOf(nullVector);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Byte value)
{
return filter.testBoolean(value != 0);
}
}

14
presto-orc/src/main/java/io/prestosql/orc/reader/BooleanSelectiveColumnReader.java
View File

@ -46,7 +46,7 @@ import static io.prestosql.spi.type.BooleanType.BOOLEAN;
import static java.util.Objects.requireNonNull;
public class BooleanSelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<Byte>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(BooleanSelectiveColumnReader.class).instanceSize();
@ -84,7 +84,7 @@ public class BooleanSelectiveColumnReader
LocalMemoryContext systemMemoryContext)
{
requireNonNull(filter, "filter is null");
checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false");
//checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false");
this.columnDesc = requireNonNull(columnDesc, "stream is null");
this.filter = filter.orElse(null);
this.outputRequired = outputRequired;
@ -158,7 +158,7 @@ public class BooleanSelectiveColumnReader
}
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
if (!rowGroupOpen) {
@ -171,7 +171,7 @@ public class BooleanSelectiveColumnReader
ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null);
}
if (filter != null) {
if (this.filter != null) {
ensureOutputPositionsCapacity(positionCount);
}
else {
@ -188,7 +188,7 @@ public class BooleanSelectiveColumnReader
if (dataStream == null && presentStream != null) {
streamPosition = readAllNulls(positions, positionCount);
}
else if (filter == null) {
else if (this.filter == null) {
streamPosition = readNoFilter(positions, positionCount);
}
else {
@ -211,7 +211,7 @@ public class BooleanSelectiveColumnReader
}
else {
boolean value = dataStream.nextBit();
if (filter == null || filter.testBoolean(value)) {
if (this.filter == null || this.filter.testBoolean(value)) {
if (outputRequired) {
values[outputPositionCount] = (byte) (value ? 1 : 0);
if (nullsAllowed && presentStream != null) {
@ -440,7 +440,7 @@ public class BooleanSelectiveColumnReader
}
@Override
public Block mergeBlocks(List<Block> blocks, int positionCount)
public Block<Byte> mergeBlocks(List<Block<Byte>> blocks, int positionCount)
{
byte[] valuesCopy = new byte[positionCount];
boolean[] nullsCopy = new boolean[positionCount];

7
presto-orc/src/main/java/io/prestosql/orc/reader/ByteColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.stream.BooleanInputStream;
@ -225,4 +226,10 @@ public class ByteColumnReader
{
return INSTANCE_SIZE + sizeOf(nullVector);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Byte value)
{
return filter.testLong(value);
}
}

8
presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReader.java
View File

@ -13,15 +13,21 @@
*/
package io.prestosql.orc.reader;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.spi.block.Block;
import java.io.IOException;
public interface ColumnReader<T>
extends AbstractColumnReader
extends AbstractColumnReader //Fixme(nitin): merge
{
Block<T> readBlock()
throws IOException;
void prepareNextRead(int batchSize);
default boolean filterTest(TupleDomainFilter filter, T value) /* FixMe(Nitin): Remove Default, force all readers to enforce */
{
throw new IllegalArgumentException("Unsupported type for pushdown");
}
}

175
presto-orc/src/main/java/io/prestosql/orc/reader/DataCachingSelectiveColumnReader.java Normal file
View File

@ -0,0 +1,175 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.orc.reader;
import com.google.common.cache.Cache;
import io.airlift.log.Logger;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcDataSourceId;
import io.prestosql.orc.OrcRowDataCacheKey;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.metadata.OrcColumnId;
import io.prestosql.orc.stream.InputStreamSources;
import io.prestosql.orc.stream.StreamSourceMeta;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.DictionaryBlock;
import java.io.IOException;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
import java.util.concurrent.ExecutionException;
public class DataCachingSelectiveColumnReader<T>
implements SelectiveColumnReader<T>
{
private static final Logger log = Logger.get(CachingColumnReader.class);
private final Cache<OrcRowDataCacheKey, Block> cache;
private final ColumnReader delegate;
private final OrcColumn column;
private final OrcColumnId columnId;
private OrcDataSourceId orcDataSourceId;
private long stripeOffset;
private long rowGroupOffset;
private int[] positions;
private int positionCount;
private Block cachedBlock;
private Block resultBlock;
private boolean isDictionary;
public DataCachingSelectiveColumnReader(ColumnReader delegate, OrcColumn column,
Cache<OrcRowDataCacheKey, Block> cache)
{
this.delegate = delegate;
this.column = column;
this.columnId = column.getColumnId();
this.cache = cache;
}
@Override
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException
{
Block block = cachedBlock.getRegion(offset, positions[positionCount - 1] + 1);
this.isDictionary = false;
this.resultBlock = block;
if (filter != null) {
this.positions = new int[positionCount];
/* traverse block and find matches */
this.positionCount = block.filter(positions, positionCount, this.positions, (value) -> delegate.filterTest(filter, value));
if (this.positionCount != positionCount) {
this.positions = Arrays.copyOf(this.positions, this.positionCount); /* since dictionary block uses arr.length */
this.isDictionary = true;
}
}
else {
this.positions = positions.clone();
this.positionCount = positionCount;
}
return this.positionCount;
}
@Override
public int readOr(int offset, int[] positions, int positionCount, List<TupleDomainFilter> filter, BitSet accumulator)
{
Block block = cachedBlock.getRegion(offset, positions[positionCount - 1] + 1);
this.positionCount = positionCount;
this.positions = positions.clone();
this.isDictionary = false;
/* traverse block and find matches */
for (int i = 0; i < positionCount; i++) {
if (accumulator.get(positions[i])
|| delegate.filterTest(filter.get(0), block.get(positions[i]))) {
accumulator.set(positions[i]);
}
}
this.resultBlock = block;
return this.positionCount;
}
@Override
public int[] getReadPositions()
{
return positions;
}
@Override
public Block getBlock(int[] positions, int positionCount)
{
if (this.resultBlock.getPositionCount() != positionCount || this.isDictionary) {
return new DictionaryBlock<T>(this.resultBlock, positions);
}
return this.resultBlock;
}
@Override
public void startStripe(ZoneId fileTimeZone, ZoneId storageTimeZone, InputStreamSources dictionaryStreamSources, ColumnMetadata<ColumnEncoding> encoding) throws IOException
{
delegate.startStripe(fileTimeZone, storageTimeZone, dictionaryStreamSources, encoding);
}
@Override
public void startRowGroup(InputStreamSources dataStreamSources) throws IOException
{
StreamSourceMeta streamSourceMeta = dataStreamSources.getStreamSourceMeta();
orcDataSourceId = streamSourceMeta.getDataSourceId();
stripeOffset = streamSourceMeta.getStripeOffset();
rowGroupOffset = streamSourceMeta.getRowGroupOffset();
cachedBlock = getCachedBlock(streamSourceMeta.getRowCount(), dataStreamSources);
//reset the stream - may not be required at all
delegate.startRowGroup(dataStreamSources);
}
@Override
public void close()
{
delegate.close();
}
@Override
public long getRetainedSizeInBytes()
{
return delegate.getRetainedSizeInBytes();
}
private Block getCachedBlock(long rowCount, InputStreamSources dataStreamSources) throws IOException
{
OrcRowDataCacheKey cacheKey = new OrcRowDataCacheKey();
cacheKey.setOrcDataSourceId(orcDataSourceId);
cacheKey.setStripeOffset(stripeOffset);
cacheKey.setRowGroupOffset(rowGroupOffset);
cacheKey.setColumnId(columnId);
try {
return cache.get(cacheKey, () -> {
delegate.startRowGroup(dataStreamSources);
delegate.prepareNextRead((int) rowCount);
log.debug("Caching row group data. DatasourceId = %s, columnId = %s, stripeOffset = %d, rowGroupOffset = %d, Column = %s", orcDataSourceId, columnId, stripeOffset, rowGroupOffset, column);
return delegate.readBlock();
});
}
catch (ExecutionException e) {
throw new IOException(e.getCause());
}
}
}

7
presto-orc/src/main/java/io/prestosql/orc/reader/DateColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.RunLengthEncodedBlock;
import io.prestosql.spi.type.DateType;
@ -92,4 +93,10 @@ public class DateColumnReader
return block;
}
@Override
public boolean filterTest(TupleDomainFilter filter, Integer value)
{
return filter.testLong(value);
}
}

12
presto-orc/src/main/java/io/prestosql/orc/reader/DecimalColumnReader.java
View File

@ -18,6 +18,7 @@ import io.airlift.slice.Slices;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.stream.BooleanInputStream;
@ -360,4 +361,15 @@ public class DecimalColumnReader<T>
{
return INSTANCE_SIZE + sizeOf(nullVector) + sizeOf(scaleVector);
}
@Override
public boolean filterTest(TupleDomainFilter filter, T value)
{
if (type.isShort()) {
return filter.testLong((Long) value);
}
long[] data = (long[]) value;
return filter.testDecimal(data[0], data[1]);
}
}

11
presto-orc/src/main/java/io/prestosql/orc/reader/DoubleColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.stream.BooleanInputStream;
@ -47,8 +48,8 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour
import static io.prestosql.spi.type.DoubleType.DOUBLE;
import static java.util.Objects.requireNonNull;
public class DoubleColumnReader<T>
implements ColumnReader<T>
public class DoubleColumnReader
implements ColumnReader<Long>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(DoubleColumnReader.class).instanceSize();
@ -227,4 +228,10 @@ public class DoubleColumnReader<T>
{
return INSTANCE_SIZE + sizeOf(nullVector);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Long value)
{
return filter.testDouble(Double.longBitsToDouble(value));
}
}

4
presto-orc/src/main/java/io/prestosql/orc/reader/DoubleSelectiveColumnReader.java
View File

@ -48,7 +48,7 @@ import static java.lang.Double.doubleToLongBits;
import static java.util.Objects.requireNonNull;
public class DoubleSelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<Long>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(DoubleSelectiveColumnReader.class).instanceSize();
@ -368,7 +368,7 @@ public class DoubleSelectiveColumnReader
}
@Override
public Block getBlock(int[] positions, int positionCount)
public Block<Long> getBlock(int[] positions, int positionCount)
{
checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
checkState(outputRequired, "This stream reader doesn't produce output");

11
presto-orc/src/main/java/io/prestosql/orc/reader/FloatColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnMetadata;
import io.prestosql.orc.stream.BooleanInputStream;
@ -46,8 +47,8 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour
import static io.prestosql.spi.type.RealType.REAL;
import static java.util.Objects.requireNonNull;
public class FloatColumnReader<T>
implements ColumnReader<T>
public class FloatColumnReader
implements ColumnReader<Integer>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(FloatColumnReader.class).instanceSize();
@ -226,4 +227,10 @@ public class FloatColumnReader<T>
{
return INSTANCE_SIZE + sizeOf(nullVector);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Integer value)
{
return filter.testFloat(Float.intBitsToFloat(value));
}
}

2
presto-orc/src/main/java/io/prestosql/orc/reader/FloatSelectiveColumnReader.java
View File

@ -46,7 +46,7 @@ import static java.lang.Float.floatToRawIntBits;
import static java.util.Objects.requireNonNull;
public class FloatSelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<Integer>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(FloatSelectiveColumnReader.class).instanceSize();
private static final Block NULL_BLOCK = REAL.createBlockBuilder(null, 1).appendNull().build();

9
presto-orc/src/main/java/io/prestosql/orc/reader/IntegerColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.IntArrayBlock;
import io.prestosql.spi.block.RunLengthEncodedBlock;
@ -50,7 +51,7 @@ public class IntegerColumnReader
}
@Override
public Block<Integer> readBlock()
public Block readBlock()
throws IOException
{
if (!rowGroupOpen) {
@ -133,4 +134,10 @@ public class IntegerColumnReader
return new IntArrayBlock(nextBatchSize, Optional.of(isNull), result);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Integer value)
{
return filter.testLong(value);
}
}

11
presto-orc/src/main/java/io/prestosql/orc/reader/LongColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.LongArrayBlock;
import io.prestosql.spi.block.RunLengthEncodedBlock;
@ -49,7 +50,7 @@ public class LongColumnReader
}
@Override
public Block<Long> readBlock()
public Block readBlock()
throws IOException
{
if (!rowGroupOpen) {
@ -70,7 +71,7 @@ public class LongColumnReader
}
}
Block<Long> block;
Block block;
if (dataStream == null) {
if (presentStream == null) {
throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is null but present stream is missing");
@ -132,4 +133,10 @@ public class LongColumnReader
return new LongArrayBlock(nextBatchSize, Optional.of(isNull), result);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Long value)
{
return filter.testLong(value);
}
}

4
presto-orc/src/main/java/io/prestosql/orc/reader/LongDecimalSelectiveColumnReader.java
View File

@ -33,7 +33,7 @@ import java.util.Optional;
import static io.prestosql.spi.type.UnscaledDecimal128Arithmetic.rescale;
public class LongDecimalSelectiveColumnReader
extends AbstractDecimalSelectiveColumnReader
extends AbstractDecimalSelectiveColumnReader<long[]>
{
public LongDecimalSelectiveColumnReader(
OrcType orcType,
@ -233,7 +233,7 @@ public class LongDecimalSelectiveColumnReader
}
@Override
protected Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values)
protected Block<long[]> makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values)
{
return new Int128ArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values);
}

10
presto-orc/src/main/java/io/prestosql/orc/reader/LongSelectiveColumnReader.java
View File

@ -44,7 +44,7 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour
import static java.util.Objects.requireNonNull;
public class LongSelectiveColumnReader
extends AbstractLongSelectiveColumnReader
extends AbstractLongSelectiveColumnReader<Long>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongSelectiveColumnReader.class).instanceSize();
@ -87,7 +87,7 @@ public class LongSelectiveColumnReader
// positions: Array of position from which data to be read;
// positionCount: Number of position in the positions array.
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
if (!rowGroupOpen) {
@ -103,7 +103,7 @@ public class LongSelectiveColumnReader
// If there is filter on this column then require space to store position. Number of position can not be more
// positionCount
if (filter != null) {
if (this.filter != null) {
ensureOutputPositionsCapacity(positionCount);
}
else {
@ -137,7 +137,7 @@ public class LongSelectiveColumnReader
if (outputRequired) {
nulls[outputPositionCount] = true;
}
if (filter != null) {
if (this.filter != null) {
outputPositions[outputPositionCount] = position;
}
outputPositionCount++;
@ -145,7 +145,7 @@ public class LongSelectiveColumnReader
}
else {
long value = dataStream.next();
if (filter == null || filter.testLong(value)) {
if (this.filter == null || this.filter.testLong(value)) {
if (outputRequired) {
values[outputPositionCount] = value;
if (nullsAllowed && presentStream != null) {

28
presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveCachingColumnReader.java → presto-orc/src/main/java/io/prestosql/orc/reader/ResultCachingSelectiveColumnReader.java
View File

@ -36,8 +36,8 @@ import java.util.BitSet;
import java.util.List;
import java.util.function.BiConsumer;
public class SelectiveCachingColumnReader
implements SelectiveColumnReader
public class ResultCachingSelectiveColumnReader<T>
implements SelectiveColumnReader<T>
{
private static final Logger log = Logger.get(SelectiveColumnReader.class);
private final Cache<OrcRowDataCacheKey, Block> cache;
@ -52,12 +52,14 @@ public class SelectiveCachingColumnReader
private int readSize;
private int totalPositionCount;
private Block cachedBlock;
private List<Block> accumulatorBlocks;
private int[] positions;
private List<Block<T>> accumulatorBlocks;
private OrcSelectiveRowDataCacheKey cacheKey;
public SelectiveCachingColumnReader(Cache<OrcRowDataCacheKey, Block> cache,
SelectiveColumnReader delegate, OrcColumn column,
OrcPredicate predicate)
public ResultCachingSelectiveColumnReader(Cache<OrcRowDataCacheKey, Block> cache,
SelectiveColumnReader delegate, OrcColumn column,
OrcPredicate predicate)
{
this.cache = cache;
this.delegate = delegate;
@ -67,16 +69,17 @@ public class SelectiveCachingColumnReader
}
@Override
public int read(int offset, int[] positions, int positionCount) throws IOException
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException
{
if (cachedBlock != null) {
this.readSize = Integer.min(positionCount, (cachedBlock.getPositionCount() - this.offset));
this.offset += this.readSize;
this.positions = positions;
return this.readSize;
}
return delegate.read(offset, positions, positionCount);
return delegate.read(offset, positions, positionCount, null);
}
@Override
@ -86,6 +89,7 @@ public class SelectiveCachingColumnReader
if (cachedBlock != null) {
this.readSize = Integer.min(positionCount, (cachedBlock.getPositionCount() - this.offset));
this.offset += this.readSize;
this.positions = positions;
if (this.readSize > 0) {
accumulator.set(offset, offset + this.readSize);
@ -99,6 +103,10 @@ public class SelectiveCachingColumnReader
@Override
public int[] getReadPositions()
{
if (cachedBlock != null) {
return this.positions;
}
return delegate.getReadPositions();
}
@ -192,7 +200,7 @@ public class SelectiveCachingColumnReader
return delegate.getRetainedSizeInBytes();
}
private Block mergeAccumulatedBlocks(List<Block> accumulatedBlocks)
private Block mergeAccumulatedBlocks(List<Block<T>> accumulatedBlocks)
{
/* Fixme(Nitin): merge all accumulated blocks in single array */
if (accumulatedBlocks.size() > 0) {
@ -201,7 +209,7 @@ public class SelectiveCachingColumnReader
totalPositionCount);
}
return new Block() {
return new Block<T>() {
@Override
public void writePositionTo(int position, BlockBuilder blockBuilder)
{

13
presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReader.java
View File

@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
public interface SelectiveColumnReader
public interface SelectiveColumnReader<T>
extends AbstractColumnReader
{
/**
@ -30,9 +30,10 @@ public interface SelectiveColumnReader
** @param positions Monotonically increasing positions to read
** @param positionCount Number of valid positions in the positions array; may be less than the
** size of the array
** @return the number of positions that passed the filter
** @param filter
* @return the number of positions that passed the filter
**/
int read(int offset, int[] positions, int positionCount)
int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException;
/**
@ -49,15 +50,15 @@ public interface SelectiveColumnReader
** from getReadPositions()
** @param positionCount Number of valid positions in the positions array; may be less than the
**/
Block getBlock(int[] positions, int positionCount);
Block<T> getBlock(int[] positions, int positionCount);
default int readOr(int offset, int[] positions, int positionCount, List<TupleDomainFilter> filter, BitSet accumulator)
throws IOException
{
return read(offset, positions, positionCount);
return read(offset, positions, positionCount, null);
}
default Block mergeBlocks(List<Block> blocks, int positionCount)
default Block<T> mergeBlocks(List<Block<T>> blocks, int positionCount)
{
return blocks.get(0);
}

25
presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReaders.java
View File

@ -57,12 +57,14 @@ public final class SelectiveColumnReaders
return new TimestampSelectiveColumnReader(column, filter, hiveStorageTimeZone, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName()));
case DECIMAL:
if (orcType.getPrecision().get() > MAX_SHORT_PRECISION) {
throw new IllegalArgumentException("Unsupported type: LONG DECIMAL");
return new LongDecimalSelectiveColumnReader(orcType, column, filter, outputType, systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName()));
}
return new ShortDecimalSelectiveColumnReader(orcType, column, filter, outputType, systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName()));
case BYTE:
case FLOAT:
case DOUBLE:
return new DoubleSelectiveColumnReader(column, filter, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName()));
case FLOAT:
return new FloatSelectiveColumnReader(column, filter, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveColumnReader.class.getSimpleName()));
case BYTE:
case LIST:
case STRUCT:
case UNION:
@ -71,11 +73,18 @@ public final class SelectiveColumnReaders
}
}
public static SelectiveColumnReader wrapWithCachingStreamReader(SelectiveColumnReader original,
OrcColumn column,
OrcPredicate predicate,
Cache<OrcRowDataCacheKey, Block> cache)
public static SelectiveColumnReader wrapWithResultCachingStreamReader(SelectiveColumnReader original,
OrcColumn column,
OrcPredicate predicate,
Cache<OrcRowDataCacheKey, Block> cache)
{
return new SelectiveCachingColumnReader(cache, original, column, predicate);
return new ResultCachingSelectiveColumnReader(cache, original, column, predicate);
}
public static SelectiveColumnReader wrapWithDataCachingStreamReader(ColumnReader original,
OrcColumn column,
Cache<OrcRowDataCacheKey, Block> cache)
{
return new DataCachingSelectiveColumnReader(original, column, cache);
}
}

7
presto-orc/src/main/java/io/prestosql/orc/reader/ShortColumnReader.java
View File

@ -16,6 +16,7 @@ package io.prestosql.orc.reader;
import io.prestosql.memory.context.LocalMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.RunLengthEncodedBlock;
import io.prestosql.spi.block.ShortArrayBlock;
@ -133,4 +134,10 @@ public class ShortColumnReader
return new ShortArrayBlock(nextBatchSize, Optional.of(isNull), result);
}
@Override
public boolean filterTest(TupleDomainFilter filter, Short value)
{
return filter.testLong(value);
}
}

4
presto-orc/src/main/java/io/prestosql/orc/reader/ShortDecimalSelectiveColumnReader.java
View File

@ -29,7 +29,7 @@ import java.util.List;
import java.util.Optional;
public class ShortDecimalSelectiveColumnReader
extends AbstractDecimalSelectiveColumnReader
extends AbstractDecimalSelectiveColumnReader<Short>
{
public ShortDecimalSelectiveColumnReader(
OrcType orcType,
@ -216,7 +216,7 @@ public class ShortDecimalSelectiveColumnReader
}
@Override
public Block mergeBlocks(List<Block> blocks, int positionCount)
public Block<Short> mergeBlocks(List<Block<Short>> blocks, int positionCount)
{
LongArrayBlockBuilder blockBuilder = new LongArrayBlockBuilder(null, positionCount);
blocks.stream().forEach(block -> {

11
presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java
View File

@ -18,6 +18,7 @@ import io.airlift.slice.Slice;
import io.prestosql.memory.context.AggregatedMemoryContext;
import io.prestosql.orc.OrcColumn;
import io.prestosql.orc.OrcCorruptionException;
import io.prestosql.orc.TupleDomainFilter;
import io.prestosql.orc.metadata.ColumnEncoding;
import io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind;
import io.prestosql.orc.metadata.ColumnMetadata;
@ -46,8 +47,8 @@ import static io.prestosql.spi.type.Varchars.byteCount;
import static io.prestosql.spi.type.Varchars.isVarcharType;
import static java.util.Objects.requireNonNull;
public class SliceColumnReader<T>
implements ColumnReader<T>
public class SliceColumnReader
implements ColumnReader<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceColumnReader.class).instanceSize();
@ -160,4 +161,10 @@ public class SliceColumnReader<T>
{
return INSTANCE_SIZE + directReader.getRetainedSizeInBytes() + dictionaryReader.getRetainedSizeInBytes();
}
@Override
public boolean filterTest(TupleDomainFilter filter, byte[] value)
{
return filter.testBytes(value, 0, value.length);
}
}

4
presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionaryColumnReader.java
View File

@ -52,8 +52,8 @@ import static java.lang.Math.toIntExact;
import static java.util.Arrays.fill;
import static java.util.Objects.requireNonNull;
public class SliceDictionaryColumnReader<T>
implements ColumnReader<T>
public class SliceDictionaryColumnReader
implements ColumnReader<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDictionaryColumnReader.class).instanceSize();

37
presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionarySelectiveColumnReader.java
View File

@ -31,7 +31,6 @@ import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.DictionaryBlock;
import io.prestosql.spi.block.RunLengthEncodedBlock;
import io.prestosql.spi.block.VariableWidthBlock;
import io.prestosql.spi.type.Chars;
import io.prestosql.spi.type.Type;
import org.openjdk.jol.info.ClassLayout;
@ -59,7 +58,7 @@ import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
public class SliceDictionarySelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDictionarySelectiveColumnReader.class).instanceSize();
@ -120,15 +119,15 @@ public class SliceDictionarySelectiveColumnReader
this.isCharType = orcType.getOrcTypeKind() == CHAR;
this.outputRequired = outputType.isPresent();
this.orcType = orcType;
checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false");
//checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false");
}
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
return readOr(offset, positions, positionCount,
(filter == null) ? null : ImmutableList.of(filter),
(this.filter == null) ? null : ImmutableList.of(this.filter),
null);
}
@ -229,18 +228,10 @@ public class SliceDictionarySelectiveColumnReader
}
else {
int index = toIntExact(dataStream.next());
int length = stripeDictionaryLength[index];
if (true) { /* Fixme(Nitin): Why this condition here?
(accumulator != null && accumulator.get(position)) || filters == null
*/
int currentPosLength = dictionaryBlock.getSliceLength(index);
int currentPosLength = dictionaryBlock.getSliceLength(index);
if (filters.get(0).testLength(currentPosLength)) {
Slice data = dictionaryBlock.getSlice(index, 0, currentPosLength);
if (isCharType) {
data = Chars.padSpaces(data, maxCodePointCount);
}
Slice finalData = data;
if (filters == null || filters.get(0).testBytes(finalData.getBytes(), 0, length)) {
if (filters == null || filters.get(0).testBytes(data.getBytes(), 0, currentPosLength)) {
if (outputRequired) {
values[outputPositionCount] = index;
}
@ -277,19 +268,11 @@ public class SliceDictionarySelectiveColumnReader
}
else {
int index = toIntExact(dataStream.next());
int length = stripeDictionaryLength[index];
if (true) { /* Fixme(Nitin): Why this condition here?
(accumulator != null && accumulator.get(position)) || filters == null
*/
int currentPosLength = dictionaryBlock.getSliceLength(index);
int currentPosLength = dictionaryBlock.getSliceLength(index);
if (filters.get(0).testLength(currentPosLength)) {
Slice data = dictionaryBlock.getSlice(index, 0, currentPosLength);
if (isCharType) {
data = Chars.padSpaces(data, maxCodePointCount);
}
Slice finalData = data;
if ((accumulator != null && accumulator.get(position))
|| filters == null || filters.get(0).testBytes(finalData.getBytes(), 0, length)) {
|| filters == null || filters.get(0).testBytes(data.getBytes(), 0, currentPosLength)) {
if (accumulator != null) {
accumulator.set(position);
}

4
presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectColumnReader.java
View File

@ -54,8 +54,8 @@ import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
public class SliceDirectColumnReader<T>
implements ColumnReader<T>
public class SliceDirectColumnReader
implements ColumnReader<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDirectColumnReader.class).instanceSize();
private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes());

70
presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectSelectiveColumnReader.java
View File

@ -57,7 +57,7 @@ import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
public class SliceDirectSelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDirectSelectiveColumnReader.class).instanceSize();
private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes());
@ -109,15 +109,15 @@ public class SliceDirectSelectiveColumnReader
this.orcType = orcType;
this.isCharType = orcType.getOrcTypeKind() == OrcType.OrcTypeKind.CHAR;
this.maxCodePointCount = orcType.getLength().orElse(-1);
checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false");
//checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false");
}
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
return readOr(offset, positions, positionCount,
(filter == null) ? null : ImmutableList.of(filter),
(this.filter == null) ? null : ImmutableList.of(this.filter),
null);
}
@ -236,41 +236,40 @@ public class SliceDirectSelectiveColumnReader
else {
int length = lengthVector[lengthIndex];
int dataOffset = outputRequired ? offset : 0;
if (true) { /* Fixme(Nitin): Why this check is needed? */
if (true /* filters.get(0).testLength(length) */) {
if (dataStream != null) {
dataStream.skip(dataToSkip);
dataToSkip = 0;
dataStream.next(data, dataOffset, dataOffset + length);
if ((accumulator != null && accumulator.get(position))
|| filters == null || filters.get(0).testBytes(data, dataOffset, length)) {
if (accumulator != null) {
accumulator.set(position);
}
}
if (outputRequired) {
int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType);
offsets[outputPositionCount + 1] = offset + truncatedLength;
if (nullsAllowed && isNullVector != null) {
nulls[outputPositionCount] = false;
}
}
outputPositions[outputPositionCount] = position;
outputPositionCount++;
}
else {
if (outputRequired) {
offsets[outputPositionCount + 1] = offset;
if (nullsAllowed && isNullVector != null) {
nulls[outputPositionCount] = false;
if ((accumulator != null && accumulator.get(position))
|| filters == null || filters.get(0).testBytes("".getBytes(), 0, 0)) {
if (accumulator != null) {
accumulator.set(position);
}
}
/* Fixme(Rajeev): This may also need to evaluate filter condition? */
outputPositions[outputPositionCount] = position;
outputPositionCount++;
}
if (outputRequired) {
int truncatedLength = 0;
if (dataStream != null) {
truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType);
}
offsets[outputPositionCount + 1] = offset + truncatedLength;
if (nullsAllowed && isNullVector != null) {
nulls[outputPositionCount] = false;
}
}
outputPositions[outputPositionCount] = position;
outputPositionCount++;
}
else {
dataToSkip += length;
@ -316,15 +315,14 @@ public class SliceDirectSelectiveColumnReader
else {
int length = lengthVector[lengthIndex];
int dataOffset = outputRequired ? offset : 0;
if (true) { /* Fixme(Nitin): Why this check is needed? */
if (isCharType || filters.get(0).testLength(length)) {
if (dataStream != null) {
dataStream.skip(dataToSkip);
dataToSkip = 0;
dataStream.next(data, dataOffset, dataOffset + length);
if (filters == null || filters.get(0).testBytes(data, dataOffset, length)) {
int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType);
if (filters == null || filters.get(0).testBytes(data, dataOffset, truncatedLength)) {
if (outputRequired) {
int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType);
offsets[outputPositionCount + 1] = offset + truncatedLength;
if (nullsAllowed && isNullVector != null) {
nulls[outputPositionCount] = false;
@ -335,16 +333,16 @@ public class SliceDirectSelectiveColumnReader
}
}
else {
if (outputRequired) {
offsets[outputPositionCount + 1] = offset;
if (nullsAllowed && isNullVector != null) {
nulls[outputPositionCount] = false;
if (filters == null || filters.get(0).testBytes("".getBytes(), 0, 0)) {
if (outputRequired) {
offsets[outputPositionCount + 1] = offset;
if (nullsAllowed && isNullVector != null) {
nulls[outputPositionCount] = false;
}
}
outputPositions[outputPositionCount] = position;
outputPositionCount++;
}
/* Fixme(Rajeev): This may also need to evaluate filter condition? */
outputPositions[outputPositionCount] = position;
outputPositionCount++;
}
}
else {

18
presto-orc/src/main/java/io/prestosql/orc/reader/SliceSelectiveColumnReader.java
View File

@ -43,7 +43,7 @@ import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT
import static java.util.Objects.requireNonNull;
public class SliceSelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceSelectiveColumnReader.class).instanceSize();
@ -111,10 +111,10 @@ public class SliceSelectiveColumnReader
}
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
return currentReader.read(offset, positions, positionCount);
return currentReader.read(offset, positions, positionCount, null);
}
@Override
@ -135,18 +135,6 @@ public class SliceSelectiveColumnReader
return currentReader.getBlock(positions, positionCount);
}
// @Override
// public BlockLease getBlockView(int[] positions, int positionCount)
// {
// return currentReader.getBlockView(positions, positionCount);
// }
//
// @Override
// public void throwAnyError(int[] positions, int positionCount)
// {
// currentReader.throwAnyError(positions, positionCount);
// }
public static int computeTruncatedLength(Slice slice, int offset, int length, int maxCodePointCount, boolean isCharType)
{
if (isCharType) {

4
presto-orc/src/main/java/io/prestosql/orc/reader/TimestampColumnReader.java
View File

@ -49,8 +49,8 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour
import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
import static java.util.Objects.requireNonNull;
public class TimestampColumnReader<T>
implements ColumnReader<T>
public class TimestampColumnReader
implements ColumnReader<Long>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(TimestampColumnReader.class).instanceSize();

8
presto-orc/src/main/java/io/prestosql/orc/reader/TimestampSelectiveColumnReader.java
View File

@ -51,7 +51,7 @@ import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
import static java.util.Objects.requireNonNull;
public class TimestampSelectiveColumnReader
implements SelectiveColumnReader
implements SelectiveColumnReader<Long>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(TimestampSelectiveColumnReader.class).instanceSize();
private static final Block NULL_BLOCK = TIMESTAMP.createBlockBuilder(null, 1).appendNull().build();
@ -141,11 +141,11 @@ public class TimestampSelectiveColumnReader
}
@Override
public int read(int offset, int[] positions, int positionCount)
public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter)
throws IOException
{
return readOr(offset, positions, positionCount,
(filter == null) ? null : ImmutableList.of(filter),
(this.filter == null) ? null : ImmutableList.of(this.filter),
null);
}
@ -426,7 +426,7 @@ public class TimestampSelectiveColumnReader
}
@Override
public Block mergeBlocks(List<Block> blocks, int positionCount)
public Block mergeBlocks(List<Block<Long>> blocks, int positionCount)
{
LongArrayBlockBuilder blockBuilder = new LongArrayBlockBuilder(null, positionCount);
blocks.stream().forEach(block -> {

5
presto-orc/src/test/java/io/prestosql/orc/BenchmarkSelectiveColumnReaders.java
View File

@ -181,7 +181,10 @@ public class BenchmarkSelectiveColumnReaders
new OrcCacheProperties(),
Optional.empty(),
new HashMap<>(),
new ArrayList<>());
new ArrayList<>(),
false,
ImmutableMap.of(),
ImmutableMap.of());
}
}

15
presto-orc/src/test/java/io/prestosql/orc/OrcTester.java
View File

@ -559,7 +559,10 @@ public class OrcTester
new OrcCacheProperties(),
Optional.empty(),
new HashMap<>(),
null);
null,
false,
ImmutableMap.of(),
ImmutableMap.of());
}
private static List<String> makeColumnNames(int columns)
@ -611,6 +614,9 @@ public class OrcTester
return false;
}
}
else if (type == TIMESTAMP) {
return filter.testLong(((SqlTimestamp) value).getMillisUtc());
}
else if (type == VARCHAR) {
return filter.testBytes(((String) value).getBytes(), 0, ((String) value).length());
}
@ -629,6 +635,11 @@ public class OrcTester
return filter.testLong(bigDecimal.unscaledValue().longValue());
}
}
else if (type == DOUBLE) {
if (!filter.testDouble((double) value)) {
return false;
}
}
else {
fail("Unsupported type: " + type);
}
@ -1377,7 +1388,7 @@ public class OrcTester
return METADATA.getParameterizedType(StandardTypes.ARRAY, ImmutableList.of(TypeSignatureParameter.of(elementType.getTypeSignature())));
}
private static Type mapType(Type keyType, Type valueType)
public static Type mapType(Type keyType, Type valueType)
{
return METADATA.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature())));
}

80
presto-orc/src/test/java/io/prestosql/orc/TestSelectiveOrcReader.java
View File

@ -26,6 +26,7 @@ import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.SqlDate;
import io.prestosql.spi.type.SqlDecimal;
import io.prestosql.spi.type.SqlTimestamp;
import org.joda.time.DateTimeZone;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@ -37,22 +38,30 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.function.Function;
import java.util.stream.IntStream;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.cycle;
import static com.google.common.collect.Iterables.limit;
import static com.google.common.collect.Lists.newArrayList;
import static io.prestosql.orc.OrcTester.HIVE_STORAGE_TIME_ZONE;
import static io.prestosql.orc.OrcTester.mapType;
import static io.prestosql.orc.OrcTester.quickSelectiveOrcTester;
import static io.prestosql.orc.TupleDomainFilter.IS_NOT_NULL;
import static io.prestosql.orc.TupleDomainFilter.IS_NULL;
import static io.prestosql.spi.type.BigintType.BIGINT;
import static io.prestosql.spi.type.BooleanType.BOOLEAN;
import static io.prestosql.spi.type.CharType.createCharType;
import static io.prestosql.spi.type.DateType.DATE;
import static io.prestosql.spi.type.DoubleType.DOUBLE;
import static io.prestosql.spi.type.IntegerType.INTEGER;
import static io.prestosql.spi.type.SmallintType.SMALLINT;
import static io.prestosql.spi.type.TimestampType.TIMESTAMP;
import static io.prestosql.spi.type.VarcharType.VARCHAR;
import static io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf;
import static io.prestosql.testing.TestingConnectorSession.SESSION;
import static java.util.Collections.nCopies;
import static java.util.stream.Collectors.toList;
import static org.testng.Assert.assertEquals;
@ -63,6 +72,7 @@ public class TestSelectiveOrcReader
private static final DecimalType DECIMAL_TYPE_PRECISION_2 = DecimalType.createDecimalType(2, 1);
private static final DecimalType DECIMAL_TYPE_PRECISION_4 = DecimalType.createDecimalType(4, 2);
private static final DecimalType DECIMAL_TYPE_PRECISION_20 = DecimalType.createDecimalType(20, 2);
private static final CharType CHAR_10 = createCharType(10);
@BeforeClass
@ -91,7 +101,12 @@ public class TestSelectiveOrcReader
public void testLongSequenceWithHoles()
throws Exception
{
testRoundTripNumeric(skipEvery(5, intsBetween(0, 31_234)), BigintRange.of(10, 100, false));
BigintRange filters1 = BigintRange.of(10, 100, false);
BigintRange filters2 = BigintRange.of(200, 300, false);
testRoundTripNumeric(intsBetween(0, 100), TupleDomainFilter.BigintMultiRange.of(ImmutableList.of(filters1, filters2), true));
long[] values = new long[]{1, 2, 3, 4};
TupleDomainFilter.BigintValues filter3 = TupleDomainFilter.BigintValues.of(values, true);
testRoundTripNumeric(intsBetween(0, 100), filter3);
}
@Test
@ -123,6 +138,10 @@ public class TestSelectiveOrcReader
.map(SqlDate::new)
.collect(toList());
List<SqlTimestamp> timestamps = longValues.stream()
.map(timestamp -> sqlTimestampOf(timestamp & Integer.MAX_VALUE, SESSION))
.collect(toList());
tester.testRoundTrip(BIGINT, longValues, ImmutableList.of(ImmutableMap.of(0, filter)));
tester.testRoundTrip(INTEGER, intValues, ImmutableList.of(ImmutableMap.of(0, filter)));
@ -131,6 +150,8 @@ public class TestSelectiveOrcReader
tester.testRoundTrip(DATE, dateValues, ImmutableList.of(ImmutableMap.of(0, filter)));
tester.testRoundTrip(TIMESTAMP, timestamps, ImmutableList.of(ImmutableMap.of(0, filter)));
List<Integer> reversedIntValues = new ArrayList<>(intValues);
Collections.reverse(reversedIntValues);
@ -248,6 +269,24 @@ public class TestSelectiveOrcReader
tester.testRoundTrip(DECIMAL_TYPE_PRECISION_2, decimalSequence("-30", "1", 60, 2, 1), filters);
}
@Test(expectedExceptions = IllegalArgumentException.class)
public void testUnknowType()
throws Exception
{
tester.testRoundTrip(mapType(INTEGER, INTEGER), createList(1, i -> createMap(1)));
}
private static <T> List<T> createList(int size, Function<Integer, T> createElement)
{
return IntStream.range(0, size).mapToObj(createElement::apply).collect(toList());
}
private static Map<Integer, Integer> createMap(int seed)
{
int mapSize = Math.abs(seed) % 7 + 1;
return IntStream.range(0, mapSize).boxed().collect(toImmutableMap(Function.identity(), i -> i + seed));
}
private static List<SqlDecimal> decimalSequence(String start, String step, int items, int precision, int scale)
{
BigInteger decimalStep = new BigInteger(step);
@ -259,4 +298,43 @@ public class TestSelectiveOrcReader
}
return values;
}
@Test
public void testDoubleSequence()
throws Exception
{
List<Map<Integer, TupleDomainFilter>> filters = ImmutableList.of(
ImmutableMap.of(0, TupleDomainFilter.DoubleRange.of(0, false, false, 1_000, false, false, false)),
ImmutableMap.of(0, IS_NULL),
ImmutableMap.of(0, IS_NOT_NULL));
tester.testRoundTrip(DOUBLE, doubleSequence(0, 0.1, 20), filters);
}
@Test
public void testDoubleNaNInfinity()
throws Exception
{
List<Map<Integer, TupleDomainFilter>> filters = ImmutableList.of(
ImmutableMap.of(0, TupleDomainFilter.DoubleRange.of(0, false, false, 1_000, false, false, false)),
ImmutableMap.of(0, IS_NULL),
ImmutableMap.of(0, IS_NOT_NULL));
tester.testRoundTrip(DOUBLE, ImmutableList.of(1000.0, -1.0, Double.POSITIVE_INFINITY), filters);
tester.testRoundTrip(DOUBLE, ImmutableList.of(-1000.0, Double.NEGATIVE_INFINITY, 1.0), filters);
tester.testRoundTrip(DOUBLE, ImmutableList.of(0.0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), filters);
tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, -1.0, 1.0), filters);
tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, -1.0, Double.POSITIVE_INFINITY), filters);
tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, Double.NEGATIVE_INFINITY, 1.0), filters);
tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), filters);
}
private static List<Double> doubleSequence(double start, double step, int items)
{
return IntStream.range(0, items)
.mapToDouble(i -> start + i * step)
.boxed()
.collect(ImmutableList.toImmutableList());
}
}

9
presto-spi/src/main/java/io/prestosql/spi/block/Block.java
View File

@ -18,6 +18,7 @@ import io.prestosql.spi.util.BloomFilter;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
import static io.prestosql.spi.block.DictionaryId.randomDictionaryId;
@ -87,7 +88,7 @@ public interface Block<T>
/**
* Gets an object in the value at {@code position}.
*/
default <T> T getObject(int position, Class<T> clazz)
default <R> R getObject(int position, Class<R> clazz)
{
throw new UnsupportedOperationException(getClass().getName());
}
@ -333,4 +334,10 @@ public interface Block<T>
//by default we will not filter anything
return validPositions;
}
default int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
System.arraycopy(positions, positionCount, matchedPositions, positionCount, positionCount);
return positionCount;
}
}

31
presto-spi/src/main/java/io/prestosql/spi/block/BlockListBlock.java
View File

@ -21,8 +21,8 @@ import java.util.function.BiConsumer;
import static io.airlift.slice.SizeOf.sizeOf;
import static java.util.Objects.requireNonNull;
public class BlockListBlock
implements Block
public class BlockListBlock<T>
implements Block<T>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(ArrayBlock.class).instanceSize();
private final Block[] blocks;
@ -197,20 +197,22 @@ public class BlockListBlock
int blockIdx = lookupBlockForPosition(positionOffset);
Block[] newBlocks = new Block[blockCount];
int newBlockCount = 0;
int newPositionCount = length;
int newPositionCount = 0;
Block block;
for (int i = blockIdx; i < blockCount && newPositionCount > 0; i++, newBlockCount++) {
block = blocks[blockIdx];
for (int i = blockIdx; i < blockCount && length > 0; i++) {
block = blocks[i];
/* if region is within the block return the block else wrap the candidate blocks */
if (block.getPositionCount() > (positionOffset - positionMap[blockIdx]) + length) {
newBlocks[i] = block.getRegion(positionOffset - positionMap[blockIdx], length);
newPositionCount -= length;
newBlocks[newBlockCount++] = block.getRegion(positionOffset - positionMap[blockIdx], length);
newPositionCount += length;
length -= length;
}
else {
newBlocks[i] = block;
newPositionCount -= block.getPositionCount();
newBlocks[newBlockCount++] = block;
newPositionCount += block.getPositionCount();
length -= block.getPositionCount();
}
}
@ -290,10 +292,10 @@ public class BlockListBlock
}
@Override
public <T> T getObject(int position, Class<T> clazz)
public <R> R getObject(int position, Class<R> clazz)
{
int blockIdx = lookupBlockForPosition(position);
return blocks[blockIdx].getObject(position - positionMap[blockIdx], clazz);
return (R) blocks[blockIdx].getObject(position - positionMap[blockIdx], clazz);
}
@Override
@ -311,4 +313,11 @@ public class BlockListBlock
return blocks[blockIdx].bytesCompare(position - positionMap[blockIdx], offset, length,
otherSlice, otherOffset, otherLength);
}
@Override
public T get(int position)
{
int blockIdx = lookupBlockForPosition(position);
return (T) blocks[blockIdx].get(position - positionMap[blockIdx]);
}
}

33
presto-spi/src/main/java/io/prestosql/spi/block/ByteArrayBlock.java
View File

@ -20,6 +20,7 @@ import javax.annotation.Nullable;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
@ -27,8 +28,8 @@ import static io.prestosql.spi.block.BlockUtil.checkValidRegion;
import static io.prestosql.spi.block.BlockUtil.compactArray;
import static io.prestosql.spi.block.BlockUtil.countUsedPositions;
public class ByteArrayBlock<T>
implements Block<T>
public class ByteArrayBlock
implements Block<Byte>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(ByteArrayBlock.class).instanceSize();
@ -233,4 +234,32 @@ public class ByteArrayBlock<T>
}
return validPositions;
}
@Override
public int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
checkReadablePosition(positions[positionCount - 1]);
int matchCount = 0;
for (int i = 0; i < positionCount; i++) {
if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) {
continue;
}
if (test.apply(values[positions[i] + arrayOffset])) {
matchedPositions[matchCount++] = positions[i];
}
}
return matchCount;
}
@Override
public Byte get(int position)
{
if (valueIsNull != null && valueIsNull[position + arrayOffset]) {
return null;
}
return values[position + arrayOffset];
}
}

35
presto-spi/src/main/java/io/prestosql/spi/block/Int128ArrayBlock.java
View File

@ -22,6 +22,7 @@ import javax.annotation.Nullable;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
@ -30,7 +31,7 @@ import static io.prestosql.spi.block.BlockUtil.compactArray;
import static io.prestosql.spi.block.BlockUtil.countUsedPositions;
public class Int128ArrayBlock
implements Block<Long>
implements Block<long[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(Int128ArrayBlock.class).instanceSize();
public static final int INT128_BYTES = Long.BYTES + Long.BYTES;
@ -244,4 +245,36 @@ public class Int128ArrayBlock
}
return validPositions;
}
@Override
public int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
int matchCount = 0;
long[] val = new long[2];
for (int i = 0; i < positionCount; i++) {
if (valueIsNull != null && valueIsNull[positions[i] + positionOffset]) {
continue;
}
val[0] = values[(positions[i] + positionOffset) * 2];
val[1] = values[((positions[i] + positionOffset) * 2) + 1];
if (test.apply(val)) {
matchedPositions[matchCount++] = positions[i];
}
}
return matchCount;
}
@Override
public long[] get(int position)
{
long[] val = new long[2];
if (valueIsNull != null && valueIsNull[position + positionOffset]) {
return null;
}
val[0] = values[(position + positionOffset) * 2];
val[1] = values[((position + positionOffset) * 2) + 1];
return val;
}
}

27
presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java
View File

@ -20,6 +20,7 @@ import javax.annotation.Nullable;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
@ -239,4 +240,30 @@ public class IntArrayBlock
}
return validPositions;
}
@Override
public int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
int matchCount = 0;
for (int i = 0; i < positionCount; i++) {
if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) {
continue;
}
if (test.apply(values[positions[i] + arrayOffset])) {
matchedPositions[matchCount++] = positions[i];
}
}
return matchCount;
}
@Override
public Integer get(int position)
{
if (valueIsNull != null && valueIsNull[position + arrayOffset]) {
return null;
}
return values[position + arrayOffset];
}
}

22
presto-spi/src/main/java/io/prestosql/spi/block/LongArrayBlock.java
View File

@ -20,6 +20,7 @@ import javax.annotation.Nullable;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
@ -133,6 +134,9 @@ public class LongArrayBlock
public Long get(int position)
{
if (valueIsNull != null && valueIsNull[position + arrayOffset]) {
return null;
}
return values[position + arrayOffset];
}
@ -283,9 +287,25 @@ public class LongArrayBlock
@Override
public boolean[] filter(BloomFilter filter, boolean[] validPositions)
{
for (int i = 0; i < values.length; i++) {
for (int i = arrayOffset; i < positionCount; i++) {
validPositions[i] = validPositions[i] && filter.test(values[i]);
}
return validPositions;
}
@Override
public int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
int matchCount = 0;
for (int i = 0; i < positionCount; i++) {
if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) {
continue;
}
if (test.apply(values[positions[i] + arrayOffset])) {
matchedPositions[matchCount++] = positions[i];
}
}
return matchCount;
}
}

27
presto-spi/src/main/java/io/prestosql/spi/block/ShortArrayBlock.java
View File

@ -20,6 +20,7 @@ import javax.annotation.Nullable;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
@ -233,4 +234,30 @@ public class ShortArrayBlock
}
return validPositions;
}
@Override
public int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
int matchCount = 0;
for (int i = 0; i < positionCount; i++) {
if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) {
continue;
}
if (test.apply(values[positions[i] + arrayOffset])) {
matchedPositions[matchCount++] = positions[i];
}
}
return matchCount;
}
@Override
public Short get(int position)
{
if (valueIsNull != null && valueIsNull[position + arrayOffset]) {
return null;
}
return values[position + arrayOffset];
}
}

29
presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlock.java
View File

@ -23,6 +23,7 @@ import javax.annotation.Nullable;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import static io.airlift.slice.SizeOf.sizeOf;
import static io.prestosql.spi.block.BlockUtil.checkArrayRange;
@ -31,8 +32,8 @@ import static io.prestosql.spi.block.BlockUtil.compactArray;
import static io.prestosql.spi.block.BlockUtil.compactOffsets;
import static io.prestosql.spi.block.BlockUtil.compactSlice;
public class VariableWidthBlock<T>
extends AbstractVariableWidthBlock<T>
public class VariableWidthBlock
extends AbstractVariableWidthBlock<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(VariableWidthBlock.class).instanceSize();
@ -234,4 +235,28 @@ public class VariableWidthBlock<T>
}
return validPositions;
}
@Override
public int filter(int[] positions, int positionCount, int[] matchedPositions, Function<Object, Boolean> test)
{
int matchCount = 0;
for (int i = 0; i < positionCount; i++) {
if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) {
continue;
}
byte[] value = slice.slice(offsets[i + arrayOffset], offsets[i + arrayOffset + 1] - offsets[i + arrayOffset]).getBytes();
if (test.apply(value)) {
matchedPositions[matchCount++] = positions[i];
}
}
return matchCount;
}
@Override
public byte[] get(int position)
{
return slice.slice(offsets[position + arrayOffset], offsets[position + arrayOffset + 1] - offsets[position + arrayOffset]).getBytes();
}
}

6
presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlockBuilder.java
View File

@ -41,9 +41,9 @@ import static io.prestosql.spi.block.BlockUtil.compactOffsets;
import static io.prestosql.spi.block.BlockUtil.compactSlice;
import static java.lang.Math.min;
public class VariableWidthBlockBuilder<T>
extends AbstractVariableWidthBlock<T>
implements BlockBuilder<T>
public class VariableWidthBlockBuilder
extends AbstractVariableWidthBlock<byte[]>
implements BlockBuilder<byte[]>
{
private static final int INSTANCE_SIZE = ClassLayout.parseClass(VariableWidthBlockBuilder.class).instanceSize();

8
presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorTableHandle.java
View File

@ -13,8 +13,11 @@
*/
package io.prestosql.spi.connector;
import io.prestosql.spi.predicate.Domain;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import java.util.function.Function;
public interface ConnectorTableHandle
{
default boolean isFilterSupported()
@ -90,4 +93,9 @@ public interface ConnectorTableHandle
{
return false;
}
default String getAdditionalFilterConditions(Function<Domain, String> printer)
{
return "";
}
}

12
presto-spi/src/main/java/io/prestosql/spi/connector/Constraint.java
View File

@ -24,7 +24,6 @@ import static java.util.Objects.requireNonNull;
public class Constraint
{
boolean isPushDownEnabled; //TODO: Rajeev: Temp,needs to be removed.
private final TupleDomain<ColumnHandle> summary;
private final Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate;
@ -55,7 +54,6 @@ public class Constraint
this.summary = summary;
this.predicate = predicate;
this.isPushDownEnabled = false;
}
public TupleDomain<ColumnHandle> getSummary()
@ -67,14 +65,4 @@ public class Constraint
{
return predicate;
}
public boolean isPushDownEnabled()
{
return isPushDownEnabled;
}
public void setPushDownEnabled(boolean pushDownEnabled)
{
isPushDownEnabled = pushDownEnabled;
}
}