diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt index d9edffeff..96861be08 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q04.plan.txt @@ -22,19 +22,37 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_29"]) - single aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_42", "c_customer_id_29", "c_email_address_44", "c_first_name_36", "c_last_name_37", "c_login_43", "c_preferred_cust_flag_38", "d_year_52"]) + partial aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_28"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["cs_sold_date_sk"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_46"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_123"]) - single aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_136", "c_customer_id_123", "c_email_address_138", "c_first_name_130", "c_last_name_131", "c_login_137", "c_preferred_cust_flag_132", "d_year_146"]) + partial aggregation over (c_birth_country_136, c_customer_id_123, c_email_address_138, c_first_name_130, c_last_name_131, c_login_137, c_preferred_cust_flag_132, d_year_146) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_122"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_140"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_247"]) final aggregation over (c_birth_country_260, c_customer_id_247, c_email_address_262, c_first_name_254, c_last_name_255, c_login_261, c_preferred_cust_flag_256, d_year_293) @@ -51,27 +69,54 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_354"]) - single aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_367", "c_customer_id_354", "c_email_address_369", "c_first_name_361", "c_last_name_362", "c_login_368", "c_preferred_cust_flag_363", "d_year_411"]) + partial aggregation over (c_birth_country_367, c_customer_id_354, c_email_address_369, c_first_name_361, c_last_name_362, c_login_368, c_preferred_cust_flag_363, d_year_411) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_353"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk_374"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["cs_sold_date_sk_371"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_405"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_482"]) - single aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_495", "c_customer_id_482", "c_email_address_497", "c_first_name_489", "c_last_name_490", "c_login_496", "c_preferred_cust_flag_491", "d_year_539"]) + partial aggregation over (c_birth_country_495, c_customer_id_482, c_email_address_497, c_first_name_489, c_last_name_490, c_login_496, c_preferred_cust_flag_491, d_year_539) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_481"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_503"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_499"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_533"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_640"]) - single aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_653", "c_customer_id_640", "c_email_address_655", "c_first_name_647", "c_last_name_648", "c_login_654", "c_preferred_cust_flag_649", "d_year_686"]) + partial aggregation over (c_birth_country_653, c_customer_id_640, c_email_address_655, c_first_name_647, c_last_name_648, c_login_654, c_preferred_cust_flag_649, d_year_686) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_639"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_660"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_657"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_680"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_747"]) final aggregation over (c_birth_country_760, c_customer_id_747, c_email_address_762, c_first_name_754, c_last_name_755, c_login_761, c_preferred_cust_flag_756, d_year_804) local exchange (GATHER, SINGLE, []) @@ -87,20 +132,38 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_875"]) - single aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_888", "c_customer_id_875", "c_email_address_890", "c_first_name_882", "c_last_name_883", "c_login_889", "c_preferred_cust_flag_884", "d_year_932"]) + partial aggregation over (c_birth_country_888, c_customer_id_875, c_email_address_890, c_first_name_882, c_last_name_883, c_login_889, c_preferred_cust_flag_884, d_year_932) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_874"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_896"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_892"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_926"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_1033"]) - single aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_1046", "c_customer_id_1033", "c_email_address_1048", "c_first_name_1040", "c_last_name_1041", "c_login_1047", "c_preferred_cust_flag_1042", "d_year_1079"]) + partial aggregation over (c_birth_country_1046, c_customer_id_1033, c_email_address_1048, c_first_name_1040, c_last_name_1041, c_login_1047, c_preferred_cust_flag_1042, d_year_1079) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_1032"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_1053"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_1050"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_1073"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_1140"]) final aggregation over (c_birth_country_1153, c_customer_id_1140, c_email_address_1155, c_first_name_1147, c_last_name_1148, c_login_1154, c_preferred_cust_flag_1149, d_year_1197) local exchange (GATHER, SINGLE, []) @@ -116,27 +179,54 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_1268"]) - single aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_1281", "c_customer_id_1268", "c_email_address_1283", "c_first_name_1275", "c_last_name_1276", "c_login_1282", "c_preferred_cust_flag_1277", "d_year_1325"]) + partial aggregation over (c_birth_country_1281, c_customer_id_1268, c_email_address_1283, c_first_name_1275, c_last_name_1276, c_login_1282, c_preferred_cust_flag_1277, d_year_1325) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_1267"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_1289"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_1285"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_1319"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_1426"]) - single aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_1439", "c_customer_id_1426", "c_email_address_1441", "c_first_name_1433", "c_last_name_1434", "c_login_1440", "c_preferred_cust_flag_1435", "d_year_1472"]) + partial aggregation over (c_birth_country_1439, c_customer_id_1426, c_email_address_1441, c_first_name_1433, c_last_name_1434, c_login_1440, c_preferred_cust_flag_1435, d_year_1472) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_1425"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_1446"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_1443"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_1466"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_1533"]) - single aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_1546", "c_customer_id_1533", "c_email_address_1548", "c_first_name_1540", "c_last_name_1541", "c_login_1547", "c_preferred_cust_flag_1542", "d_year_1590"]) + partial aggregation over (c_birth_country_1546, c_customer_id_1533, c_email_address_1548, c_first_name_1540, c_last_name_1541, c_login_1547, c_preferred_cust_flag_1542, d_year_1590) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_1532"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk_1553"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["cs_sold_date_sk_1550"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_1584"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_1661"]) final aggregation over (c_birth_country_1674, c_customer_id_1661, c_email_address_1676, c_first_name_1668, c_last_name_1669, c_login_1675, c_preferred_cust_flag_1670, d_year_1718) local exchange (GATHER, SINGLE, []) @@ -154,19 +244,37 @@ local exchange (GATHER, SINGLE, []) scan customer local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_1819"]) - single aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_1832", "c_customer_id_1819", "c_email_address_1834", "c_first_name_1826", "c_last_name_1827", "c_login_1833", "c_preferred_cust_flag_1828", "d_year_1865"]) + partial aggregation over (c_birth_country_1832, c_customer_id_1819, c_email_address_1834, c_first_name_1826, c_last_name_1827, c_login_1833, c_preferred_cust_flag_1828, d_year_1865) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_1818"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_1839"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_1836"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_1859"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_1926"]) - single aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_1939", "c_customer_id_1926", "c_email_address_1941", "c_first_name_1933", "c_last_name_1934", "c_login_1940", "c_preferred_cust_flag_1935", "d_year_1983"]) + partial aggregation over (c_birth_country_1939, c_customer_id_1926, c_email_address_1941, c_first_name_1933, c_last_name_1934, c_login_1940, c_preferred_cust_flag_1935, d_year_1983) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_1925"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["cs_bill_customer_sk_1946"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["cs_sold_date_sk_1943"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_1977"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_2054"]) final aggregation over (c_birth_country_2067, c_customer_id_2054, c_email_address_2069, c_first_name_2061, c_last_name_2062, c_login_2068, c_preferred_cust_flag_2063, d_year_2111) local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt index d6d71fad2..32b0a6bfe 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q11.plan.txt @@ -19,12 +19,21 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_29"]) - single aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_42", "c_customer_id_29", "c_email_address_44", "c_first_name_36", "c_last_name_37", "c_login_43", "c_preferred_cust_flag_38", "d_year_52"]) + partial aggregation over (c_birth_country_42, c_customer_id_29, c_email_address_44, c_first_name_36, c_last_name_37, c_login_43, c_preferred_cust_flag_38, d_year_52) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_28"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_46"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_153"]) final aggregation over (c_birth_country_166, c_customer_id_153, c_email_address_168, c_first_name_160, c_last_name_161, c_login_167, c_preferred_cust_flag_162, d_year_199) @@ -41,20 +50,38 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_260"]) - single aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_273", "c_customer_id_260", "c_email_address_275", "c_first_name_267", "c_last_name_268", "c_login_274", "c_preferred_cust_flag_269", "d_year_317"]) + partial aggregation over (c_birth_country_273, c_customer_id_260, c_email_address_275, c_first_name_267, c_last_name_268, c_login_274, c_preferred_cust_flag_269, d_year_317) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_259"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_281"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_277"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_311"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_418"]) - single aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_431", "c_customer_id_418", "c_email_address_433", "c_first_name_425", "c_last_name_426", "c_login_432", "c_preferred_cust_flag_427", "d_year_464"]) + partial aggregation over (c_birth_country_431, c_customer_id_418, c_email_address_433, c_first_name_425, c_last_name_426, c_login_432, c_preferred_cust_flag_427, d_year_464) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_417"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_438"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_435"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_458"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_525"]) final aggregation over (c_birth_country_538, c_customer_id_525, c_email_address_540, c_first_name_532, c_last_name_533, c_login_539, c_preferred_cust_flag_534, d_year_582) local exchange (GATHER, SINGLE, []) @@ -72,12 +99,21 @@ local exchange (GATHER, SINGLE, []) scan customer local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_683"]) - single aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_birth_country_696", "c_customer_id_683", "c_email_address_698", "c_first_name_690", "c_last_name_691", "c_login_697", "c_preferred_cust_flag_692", "d_year_729"]) + partial aggregation over (c_birth_country_696, c_customer_id_683, c_email_address_698, c_first_name_690, c_last_name_691, c_login_697, c_preferred_cust_flag_692, d_year_729) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_682"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_703"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_700"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_723"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_790"]) final aggregation over (c_birth_country_803, c_customer_id_790, c_email_address_805, c_first_name_797, c_last_name_798, c_login_804, c_preferred_cust_flag_799, d_year_847) local exchange (GATHER, SINGLE, []) diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt index 580389d9f..7d4804aab 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q34.plan.txt @@ -16,10 +16,10 @@ remote exchange (GATHER, SINGLE, []) scan store local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan household_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan household_demographics + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_sk"]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt index 9fa2321b1..2c20a53b7 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q73.plan.txt @@ -16,10 +16,10 @@ remote exchange (GATHER, SINGLE, []) scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store + scan household_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan household_demographics + scan store local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_sk"]) scan customer diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt index a78033681..5f8205dba 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q74.plan.txt @@ -19,12 +19,21 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_17"]) - single aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_customer_id_17", "c_first_name_24", "c_last_name_25", "d_year_40"]) + partial aggregation over (c_customer_id_17, c_first_name_24, c_last_name_25, d_year_40) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_16"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_34"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_109"]) final aggregation over (c_customer_id_109, c_first_name_116, c_last_name_117, d_year_155) @@ -41,20 +50,38 @@ local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan customer remote exchange (REPARTITION, HASH, ["c_customer_id_200"]) - single aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_customer_id_200", "c_first_name_207", "c_last_name_208", "d_year_257"]) + partial aggregation over (c_customer_id_200, c_first_name_207, c_last_name_208, d_year_257) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_199"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ws_bill_customer_sk_221"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ws_sold_date_sk_217"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_251"]) + values (0 rows) local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_326"]) - single aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_customer_id_326", "c_first_name_333", "c_last_name_334", "d_year_372"]) + partial aggregation over (c_customer_id_326, c_first_name_333, c_last_name_334, d_year_372) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_325"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_346"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_343"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_366"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_417"]) final aggregation over (c_customer_id_417, c_first_name_424, c_last_name_425, d_year_474) local exchange (GATHER, SINGLE, []) @@ -72,12 +99,21 @@ local exchange (GATHER, SINGLE, []) scan customer local exchange (GATHER, SINGLE, []) remote exchange (REPARTITION, HASH, ["c_customer_id_543"]) - single aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - values (0 rows) - values (0 rows) - values (0 rows) + final aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["c_customer_id_543", "c_first_name_550", "c_last_name_551", "d_year_589"]) + partial aggregation over (c_customer_id_543, c_first_name_550, c_last_name_551, d_year_589) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["c_customer_sk_542"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["ss_customer_sk_563"]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, ["ss_sold_date_sk_560"]) + values (0 rows) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, ["d_date_sk_583"]) + values (0 rows) remote exchange (REPARTITION, HASH, ["c_customer_id_634"]) final aggregation over (c_customer_id_634, c_first_name_641, c_last_name_642, d_year_691) local exchange (GATHER, SINGLE, []) diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java index 8c6770dd4..feafbb2a0 100755 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java @@ -1976,7 +1976,11 @@ public class HiveMetadata .orElseThrow(() -> new TableNotFoundException(hiveTable.getSchemaTableName())); if (hmsTable.getStorage().getStorageFormat().getSerDe().equalsIgnoreCase(HiveStorageFormat.ORC.getSerDe())) { - predicate = hiveTable.getEnforcedConstraint(); + ImmutableMap.Builder pushedDown = ImmutableMap.builder(); + pushedDown.putAll(hiveTable.getCompactEffectivePredicate().getDomains().get().entrySet().stream() + .collect(toMap(e -> (ColumnHandle) e.getKey(), e -> e.getValue()))); + + predicate = predicate.intersect(withColumnDomains(pushedDown.build())); } } @@ -2059,7 +2063,8 @@ public class HiveMetadata .forEach(predicateColumnNames::add); List> newEffectivePredicates = null; - if (HiveSessionProperties.isOrcPredicatePushdownEnabled(session)) { + if (HiveSessionProperties.isOrcPredicatePushdownEnabled(session) + && HiveSessionProperties.isOrcDisjunctPredicatePushdownEnabled(session)) { newEffectivePredicates = builder.build(); newEffectivePredicates.stream().forEach(nfp -> diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java index e29e72774..0875f377a 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java @@ -41,6 +41,7 @@ import org.joda.time.DateTimeZone; import javax.inject.Inject; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -132,12 +133,20 @@ public class HivePageSourceProvider Configuration configuration = hdfsEnvironment.getConfiguration( new HdfsEnvironment.HdfsContext(session, hiveSplit.getDatabase(), hiveSplit.getTable()), path); - List indexes = null; + List indexes = new ArrayList<>(); if (indexCache != null && session.isHeuristicIndexFilterEnabled()) { - indexes = indexCache.getIndices( - session.getCatalog().orElse(null), - hiveTable.getSchemaTableName().toString(), hiveSplit, hiveTable.getCompactEffectivePredicate(), - hiveTable.getPartitionColumns()); + indexes.addAll(this.indexCache.getIndices(session + .getCatalog().orElse(null), hiveTable + .getSchemaTableName().toString(), hiveSplit, hiveTable.getCompactEffectivePredicate(), + hiveTable.getPartitionColumns())); + + if (hiveTable.getAdditionalCompactEffectivePredicate().isPresent() && hiveTable.getAdditionalCompactEffectivePredicate().get().size() > 0) { + hiveTable.getAdditionalCompactEffectivePredicate().get().forEach(orPredicate -> + indexes.addAll(this.indexCache.getIndices(session + .getCatalog().orElse(null), hiveTable + .getSchemaTableName().toString(), hiveSplit, orPredicate, hiveTable + .getPartitionColumns()))); + } } Optional> indexOptional = indexes == null || indexes.isEmpty() ? Optional.empty() : Optional.of(indexes); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSelectivePageSourceFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSelectivePageSourceFactory.java index 9b79cc5f8..3afd720a3 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSelectivePageSourceFactory.java @@ -13,10 +13,9 @@ */ package io.prestosql.plugin.hive; -import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.plugin.hive.coercions.HiveCoercer; import io.prestosql.spi.connector.ConnectorPageSource; import io.prestosql.spi.connector.ConnectorSession; -import io.prestosql.spi.dynamicfilter.DynamicFilter; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.predicate.TupleDomain; import org.apache.hadoop.conf.Configuration; @@ -44,9 +43,10 @@ public interface HiveSelectivePageSourceFactory TupleDomain domainPredicate, Optional>> additionPredicates, DateTimeZone hiveStorageTimeZone, - Map dynamicFilter, Optional deleteDeltaLocations, Optional startRowOffsetOfFile, Optional> indexes, - boolean splitCacheable); + boolean splitCacheable, + List columnMappings, + Map coercers); } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java index 48d850f49..c20ca9ad0 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSessionProperties.java @@ -85,8 +85,10 @@ public final class HiveSessionProperties private static final String TEMPORARY_STAGING_DIRECTORY_ENABLED = "temporary_staging_directory_enabled"; private static final String TEMPORARY_STAGING_DIRECTORY_PATH = "temporary_staging_directory_path"; private static final String DYNAMIC_FILTERING_SPLIT_FILTERING = "dynamic_filtering_partition_filtering"; - private static final String ORC_PREDICATE_PUSHDOWN = "orc_predicate_pushdown_enabled"; private static final String DYNAMIC_FILTERING_ROW_FILTERING_THRESHOLD = "dynamic_filtering_filter_rows_threshold"; + private static final String ORC_PREDICATE_PUSHDOWN = "orc_predicate_pushdown_enabled"; + private static final String ORC_DISJUCT_PREDICATE_PUSHDOWN = "orc_disjunct_predicate_pushdown_enabled"; + private static final String ORC_PUSHDOWN_DATACACHE = "orc_pushdown_data_cache_enabled"; private final List> sessionProperties; @@ -365,8 +367,18 @@ public final class HiveSessionProperties false), booleanProperty( ORC_PREDICATE_PUSHDOWN, - "Experimental: Consume deterministic predicates for ORC scan.", + "Experimental: Consume deterministic predicates(conjucts: AND) for ORC scan.", false, + false), + booleanProperty( + ORC_DISJUCT_PREDICATE_PUSHDOWN, + "Experimental: Consume deterministic predicates(disjucts: OR) for ORC scan.", + true, + false), + booleanProperty( + ORC_PUSHDOWN_DATACACHE, + "Experimental: Enable data cache or result cache with predicate pushdown.", + true, false)); } @@ -621,4 +633,14 @@ public final class HiveSessionProperties { return session.getProperty(ORC_PREDICATE_PUSHDOWN, Boolean.class); } + + public static boolean isOrcDisjunctPredicatePushdownEnabled(ConnectorSession session) + { + return session.getProperty(ORC_DISJUCT_PREDICATE_PUSHDOWN, Boolean.class); + } + + public static boolean isOrcPushdownDataCacheEnabled(ConnectorSession session) + { + return session.getProperty(ORC_PUSHDOWN_DATACACHE, Boolean.class); + } } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java index ee3c0cffa..5f02dd229 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveSplitSource.java @@ -134,6 +134,7 @@ class HiveSplitSource this.maxSplitSize = getMaxSplitSize(session); this.maxInitialSplitSize = getMaxInitialSplitSize(session); this.remainingInitialSplits = new AtomicInteger(maxInitialSplits); + this.dynamicFilterSupplier = dynamicFilterSupplier; this.isSplitFilteringEnabled = isDynamicFilteringSplitFilteringEnabled(session); this.userDefinedCachePredicates = userDefinedCachedPredicates; diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveTableHandle.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveTableHandle.java index 530810cfe..ae9c13029 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveTableHandle.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveTableHandle.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ConnectorTableHandle; import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.predicate.Domain; import io.prestosql.spi.predicate.TupleDomain; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -28,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; @@ -274,6 +277,25 @@ public class HiveTableHandle && additionalCompactEffectivePredicate.get().size() > 0; } + private String formatPredicate(Function printer, TupleDomain predicate) + { + return predicate.getDomains().get().entrySet().stream() + .map(filter -> filter.getKey().getColumnName() + " <- " + printer.apply(filter.getValue())) + .collect(Collectors.joining(" AND ", "{", "}")); + } + + @Override + public String getAdditionalFilterConditions(Function printer) + { + if (additionalCompactEffectivePredicate.isPresent()) { + return additionalCompactEffectivePredicate.get().stream() + .map(predicate -> "[ " + formatPredicate(printer, predicate) + " ]") + .collect(Collectors.joining(" OR ")); + } + + return ""; + } + @Override public boolean equals(Object o) { diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriteUtils.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriteUtils.java index 830721049..ab36a0df8 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveWriteUtils.java @@ -326,7 +326,7 @@ public final class HiveWriteUtils if (isArrayType(type)) { Type elementType = type.getTypeParameters().get(0); - Block arrayBlock = block.getObject(position, Block.class); + Block arrayBlock = block.getObject(position, Block.class); List list = new ArrayList<>(arrayBlock.getPositionCount()); for (int i = 0; i < arrayBlock.getPositionCount(); i++) { @@ -1093,7 +1093,7 @@ public final class HiveWriteUtils @Override public void setField(Block block, int position) { - Block mapBlock = block.getObject(position, Block.class); + Block mapBlock = block.getObject(position, Block.class); Map map = new HashMap<>(mapBlock.getPositionCount() * 2); for (int i = 0; i < mapBlock.getPositionCount(); i += 2) { Object key = getField(keyType, mapBlock, i); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/coercions/DecimalCoercers.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/coercions/DecimalCoercers.java index b5ef37187..6a8789710 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/coercions/DecimalCoercers.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/coercions/DecimalCoercers.java @@ -21,8 +21,6 @@ import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.DoubleType; import io.prestosql.spi.type.RealType; -import java.util.function.Function; - import static io.prestosql.spi.type.DecimalConversions.doubleToLongDecimal; import static io.prestosql.spi.type.DecimalConversions.doubleToShortDecimal; import static io.prestosql.spi.type.DecimalConversions.longDecimalToDouble; @@ -43,7 +41,7 @@ public final class DecimalCoercers { private DecimalCoercers() {} - public static Function createDecimalToDecimalCoercer(DecimalType fromType, DecimalType toType) + public static HiveCoercer createDecimalToDecimalCoercer(DecimalType fromType, DecimalType toType) { if (fromType.isShort()) { if (toType.isShort()) { @@ -148,7 +146,7 @@ public final class DecimalCoercers } } - public static Function createDecimalToDoubleCoercer(DecimalType fromType) + public static HiveCoercer createDecimalToDoubleCoercer(DecimalType fromType) { if (fromType.isShort()) { return new ShortDecimalToDoubleCoercer(fromType); @@ -193,7 +191,7 @@ public final class DecimalCoercers } } - public static Function createDecimalToRealCoercer(DecimalType fromType) + public static HiveCoercer createDecimalToRealCoercer(DecimalType fromType) { if (fromType.isShort()) { return new ShortDecimalToRealCoercer(fromType); @@ -238,7 +236,7 @@ public final class DecimalCoercers } } - public static Function createDoubleToDecimalCoercer(DecimalType toType) + public static HiveCoercer createDoubleToDecimalCoercer(DecimalType toType) { if (toType.isShort()) { return new DoubleToShortDecimalCoercer(toType); @@ -280,7 +278,7 @@ public final class DecimalCoercers } } - public static Function createRealToDecimalCoercer(DecimalType toType) + public static HiveCoercer createRealToDecimalCoercer(DecimalType toType) { if (toType.isShort()) { return new RealToShortDecimalCoercer(toType); diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSource.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSource.java index 85089e217..dc7418e70 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSource.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSource.java @@ -18,12 +18,18 @@ import io.prestosql.orc.OrcCorruptionException; import io.prestosql.orc.OrcDataSource; import io.prestosql.orc.OrcSelectiveRecordReader; import io.prestosql.plugin.hive.FileFormatDataSourceStats; +import io.prestosql.plugin.hive.HiveColumnHandle; +import io.prestosql.plugin.hive.HivePageSourceProvider.ColumnMapping; import io.prestosql.spi.Page; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ConnectorPageSource; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeManager; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.List; import static com.google.common.base.MoreObjects.toStringHelper; import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_BAD_DATA; @@ -40,17 +46,31 @@ public class OrcSelectivePageSource private final FileFormatDataSourceStats stats; private boolean closed; + private final Type[] types; public OrcSelectivePageSource( OrcSelectiveRecordReader recordReader, OrcDataSource orcDataSource, AggregatedMemoryContext systemMemoryContext, - FileFormatDataSourceStats stats) + FileFormatDataSourceStats stats, + ConnectorSession session, + List columnMappings, + TypeManager typeManager) { this.recordReader = requireNonNull(recordReader, "recordReader is null"); this.orcDataSource = requireNonNull(orcDataSource, "orcDataSource is null"); this.systemMemoryContext = requireNonNull(systemMemoryContext, "systemMemoryContext is null"); this.stats = requireNonNull(stats, "stats is null"); + + types = new Type[columnMappings.size()]; + for (int columnIndex = 0; columnIndex < columnMappings.size(); columnIndex++) { + ColumnMapping columnMapping = columnMappings.get(columnIndex); + HiveColumnHandle column = columnMapping.getHiveColumnHandle(); + + String name = column.getName(); + Type type = typeManager.getType(column.getTypeSignature()); + types[columnIndex] = type; + } } @Override @@ -78,7 +98,9 @@ public class OrcSelectivePageSource Page page = recordReader.getNextPage(); if (page == null) { close(); + return null; } + return page; } catch (PrestoException e) { diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSourceFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSourceFactory.java index c38e57879..5ff4f23c5 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSourceFactory.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/orc/OrcSelectivePageSourceFactory.java @@ -37,16 +37,17 @@ import io.prestosql.plugin.hive.FileFormatDataSourceStats; import io.prestosql.plugin.hive.HdfsEnvironment; import io.prestosql.plugin.hive.HiveColumnHandle; import io.prestosql.plugin.hive.HiveConfig; +import io.prestosql.plugin.hive.HivePageSourceProvider; import io.prestosql.plugin.hive.HiveSelectivePageSourceFactory; +import io.prestosql.plugin.hive.HiveSessionProperties; import io.prestosql.plugin.hive.HiveType; import io.prestosql.plugin.hive.HiveUtil; +import io.prestosql.plugin.hive.coercions.HiveCoercer; import io.prestosql.plugin.hive.orc.OrcPageSource.ColumnAdaptation; import io.prestosql.spi.PrestoException; -import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ConnectorPageSource; import io.prestosql.spi.connector.ConnectorSession; import io.prestosql.spi.connector.FixedPageSource; -import io.prestosql.spi.dynamicfilter.DynamicFilter; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.predicate.Domain; import io.prestosql.spi.predicate.TupleDomain; @@ -75,6 +76,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -157,11 +160,12 @@ public class OrcSelectivePageSourceFactory TupleDomain domainPredicate, Optional>> additionPredicates, DateTimeZone hiveStorageTimeZone, - Map dynamicFilter, Optional deleteDeltaLocations, Optional startRowOffsetOfFile, Optional> indexes, - boolean splitCacheable) + boolean splitCacheable, + List columnMappings, + Map coercers) { if (!HiveUtil.isDeserializerClass(schema, OrcSerde.class)) { return Optional.empty(); @@ -185,7 +189,7 @@ public class OrcSelectivePageSourceFactory return Optional.of(createOrcPageSource( hdfsEnvironment, - session.getUser(), + session, configuration, path, start, @@ -207,14 +211,15 @@ public class OrcSelectivePageSourceFactory getOrcLazyReadSmallRanges(session), isOrcBloomFiltersEnabled(session), stats, - dynamicFilter, deleteDeltaLocations, startRowOffsetOfFile, indexes, orcCacheStore, orcCacheProperties, additionPredicates.orElseGet(() -> ImmutableList.of()), - positions)); + positions, + columnMappings, + coercers)); /* Todo(Nitin): For Append Pattern appendPredicates.get().stream().forEach(newDomainPredicate -> @@ -242,7 +247,6 @@ public class OrcSelectivePageSourceFactory getOrcLazyReadSmallRanges(session), isOrcBloomFiltersEnabled(session), stats, - dynamicFilter, deleteDeltaLocations, startRowOffsetOfFile, indexes, @@ -258,7 +262,7 @@ public class OrcSelectivePageSourceFactory return Optional.of(createOrcPageSource( hdfsEnvironment, - session.getUser(), + session, configuration, path, start, @@ -280,19 +284,20 @@ public class OrcSelectivePageSourceFactory getOrcLazyReadSmallRanges(session), isOrcBloomFiltersEnabled(session), stats, - dynamicFilter, deleteDeltaLocations, startRowOffsetOfFile, indexes, orcCacheStore, orcCacheProperties, ImmutableList.of(), - null)); + null, + columnMappings, + coercers)); } public static OrcSelectivePageSource createOrcPageSource( HdfsEnvironment hdfsEnvironment, - String sessionUser, + ConnectorSession session, Configuration configuration, Path path, long start, @@ -314,17 +319,18 @@ public class OrcSelectivePageSourceFactory boolean lazyReadSmallRanges, boolean orcBloomFiltersEnabled, FileFormatDataSourceStats stats, - Map dynamicFilter, Optional deleteDeltaLocations, Optional startRowOffsetOfFile, Optional> indexes, OrcCacheStore orcCacheStore, OrcCacheProperties orcCacheProperties, List> additionalDomainPredicates, - List positions) + List positions, + List columnMappings, + Map coercers) { checkArgument(!domainPredicate.isNone(), "Unexpected NONE domain"); - + String sessionUser = session.getUser(); OrcDataSource orcDataSource; try { //Always create a lazy Stream. HDFS stream opened only when required. @@ -401,6 +407,7 @@ public class OrcSelectivePageSourceFactory additionalDomainPredicates.stream() .forEach(ap -> ap.getDomains().get().forEach((k, v) -> additionalPredicateDomains.merge(k, v, (v1, v2) -> v1.union(v2)))); + Map> orDomains = new ConcurrentHashMap<>(); for (HiveColumnHandle column : columns) { OrcColumn orcColumn = null; if (useOrcColumnNames || isFullAcid) { @@ -425,6 +432,7 @@ public class OrcSelectivePageSourceFactory domain = additionalPredicateDomains.get(column); if (domain != null) { predicateBuilder.addOrColumn(orcColumn.getColumnId(), domain); + orDomains.computeIfAbsent(column.getName(), l -> new ArrayList<>()).add(domain); } } else if (isFullAcid && readType instanceof RowType && column.getName().equalsIgnoreCase("row__id")) { @@ -486,7 +494,10 @@ public class OrcSelectivePageSourceFactory orcCacheProperties, Optional.empty(), orFilters, - positions); + positions, + HiveSessionProperties.isOrcPushdownDataCacheEnabled(session), + Maps.transformValues(coercers, Function.class::cast), + orDomains); OrcDeletedRows deletedRows = new OrcDeletedRows( path.getName(), @@ -508,7 +519,10 @@ public class OrcSelectivePageSourceFactory // deletedRows, // isFullAcid && indexes.isPresent(), systemMemoryUsage, - stats); + stats, + session, + columnMappings, + typeManager); } catch (Exception e) { try { diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java index 2f3455937..a32e34d5c 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/IndexCache.java @@ -92,7 +92,7 @@ public class IndexCache .map(HiveColumnHandle::getName) .map(String::toLowerCase).forEach(column -> { String indexCacheKeyPath = Paths.get(tableFqn, column, pathUri.getPath()).toString(); - IndexCacheKey indexCacheKey = new IndexCacheKey(indexCacheKeyPath, lastModifiedTime, "bitmap"); + IndexCacheKey indexCacheKey = new IndexCacheKey(indexCacheKeyPath, lastModifiedTime, "bitmap", "bloom"); // check if cache contains the key List predicateIndexes = cache.getIfPresent(indexCacheKey); diff --git a/presto-main/src/main/java/io/prestosql/query/CachedSqlQueryExecution.java b/presto-main/src/main/java/io/prestosql/query/CachedSqlQueryExecution.java index df3c54d0a..dbf791012 100644 --- a/presto-main/src/main/java/io/prestosql/query/CachedSqlQueryExecution.java +++ b/presto-main/src/main/java/io/prestosql/query/CachedSqlQueryExecution.java @@ -372,7 +372,8 @@ public class CachedSqlQueryExecution connectorTransactionHandleMap.put(tableHandle.getTransaction(), newTableHandle.getTransaction()); // Return a new table handle with the ID, output symbols, assignments, and enforced constraints of the cached table handle - return new TableScanNode(node.getId(), newTableHandle, node.getOutputSymbols(), node.getAssignments(), node.getEnforcedConstraint()); + return new TableScanNode(node.getId(), newTableHandle, node.getOutputSymbols(), node.getAssignments(), + node.getEnforcedConstraint(), node.getPredicate()); } @Override diff --git a/presto-main/src/main/java/io/prestosql/sql/builder/optimizer/SubQueryPushDown.java b/presto-main/src/main/java/io/prestosql/sql/builder/optimizer/SubQueryPushDown.java index 2547e71f3..8aef33ec8 100644 --- a/presto-main/src/main/java/io/prestosql/sql/builder/optimizer/SubQueryPushDown.java +++ b/presto-main/src/main/java/io/prestosql/sql/builder/optimizer/SubQueryPushDown.java @@ -316,7 +316,8 @@ public class SubQueryPushDown result.get().getHandle(), symbolsBuilder.build(), columnHandleBuilder.build(), - TupleDomain.all()), + TupleDomain.all(), + Optional.empty()), new Assignments(assignmentsBuilder.build())); return Optional.of(output); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java b/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java index 505861508..93e605775 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/PlanFragmenter.java @@ -795,7 +795,8 @@ public class PlanFragmenter newTable, node.getOutputSymbols(), node.getAssignments(), - node.getEnforcedConstraint()); + node.getEnforcedConstraint(), + node.getPredicate()); } } } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java index 9866796c2..ee07252dd 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/PlanOptimizers.java @@ -422,7 +422,7 @@ public class PlanOptimizers ImmutableSet.>builder() .addAll(projectionPushdownRules) .add(new PushLimitIntoTableScan(metadata)) - .add(new PushPredicateIntoTableScan(metadata, typeAnalyzer)) + //.add(new PushPredicateIntoTableScan(metadata, typeAnalyzer)) .add(new PushSampleIntoTableScan(metadata)) .build()), new IterativeOptimizer( @@ -477,11 +477,11 @@ public class PlanOptimizers ImmutableSet.of(new EliminateCrossJoins())), // This can pull up Filter and Project nodes from between Joins, so we need to push them down again predicatePushDown, simplifyOptimizer, // Should be always run after PredicatePushDown - new IterativeOptimizer( + /*new IterativeOptimizer( ruleStats, statsCalculator, estimatedExchangesCostCalculator, - ImmutableSet.of(new PushPredicateIntoTableScan(metadata, typeAnalyzer))), + ImmutableSet.of(new PushPredicateIntoTableScan(metadata, typeAnalyzer))),*/ projectionPushDown, new PruneUnreferencedOutputs(), new IterativeOptimizer( @@ -524,12 +524,6 @@ public class PlanOptimizers .add(new InlineProjections()) .build())); - builder.add(new IterativeOptimizer( - ruleStats, - statsCalculator, - costCalculator, - ImmutableSet.of(new PushDeleteIntoConnector(metadata)))); // Must run before AddExchanges - if (!forceSingleNode) { builder.add(new ReplicateSemiJoinInDelete()); // Must run before AddExchanges builder.add((new IterativeOptimizer( @@ -549,6 +543,23 @@ public class PlanOptimizers ImmutableSet.of(new PushTableWriteThroughUnion()))); // Must run before AddExchanges builder.add(new StatsRecordingPlanOptimizer(optimizerStats, new AddExchanges(metadata, typeAnalyzer))); } + + IterativeOptimizer pushdownRule = new IterativeOptimizer( + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + ImmutableSet.of(new PushPredicateIntoTableScan(metadata, typeAnalyzer))); + + IterativeOptimizer pushdownDeleteRule = new IterativeOptimizer( + ruleStats, + statsCalculator, + costCalculator, + ImmutableSet.of(new PushDeleteIntoConnector(metadata, false))); // Must run before AddExchanges + IterativeOptimizer pushdownDeleteWithExchangeRule = new IterativeOptimizer( + ruleStats, + statsCalculator, + costCalculator, + ImmutableSet.of(new PushDeleteIntoConnector(metadata, true))); // Must run before AddExchanges //noinspection UnusedAssignment estimatedExchangesCostCalculator = null; // Prevent accidental use after AddExchanges @@ -576,6 +587,9 @@ public class PlanOptimizers .add(new PushRemoteExchangeThroughAssignUniqueId()) .add(new InlineProjections()) .build())); + builder.add(pushdownRule); + builder.add(pushdownDeleteWithExchangeRule); + builder.add(pushdownDeleteRule); // Optimizers above this don't understand local exchanges, so be careful moving this. builder.add(new AddLocalExchanges(metadata, typeAnalyzer)); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PruneTableScanColumns.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PruneTableScanColumns.java index e2c7d27e3..c32420c44 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PruneTableScanColumns.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PruneTableScanColumns.java @@ -42,6 +42,7 @@ public class PruneTableScanColumns tableScanNode.getTable(), filteredCopy(tableScanNode.getOutputSymbols(), referencedOutputs::contains), filterKeys(tableScanNode.getAssignments(), referencedOutputs::contains), - tableScanNode.getEnforcedConstraint())); + tableScanNode.getEnforcedConstraint(), + tableScanNode.getPredicate())); } } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushDeleteIntoConnector.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushDeleteIntoConnector.java index 56dff1d6c..cd214015d 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushDeleteIntoConnector.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushDeleteIntoConnector.java @@ -25,6 +25,7 @@ import io.prestosql.sql.planner.plan.TableScanNode; import static com.google.common.collect.Iterables.getOnlyElement; import static io.prestosql.matching.Capture.newCapture; import static io.prestosql.sql.planner.plan.Patterns.delete; +import static io.prestosql.sql.planner.plan.Patterns.exchange; import static io.prestosql.sql.planner.plan.Patterns.source; import static io.prestosql.sql.planner.plan.Patterns.tableFinish; import static io.prestosql.sql.planner.plan.Patterns.tableScan; @@ -34,22 +35,34 @@ public class PushDeleteIntoConnector implements Rule { private static final Capture TABLE_SCAN = newCapture(); - private static final Pattern PATTERN = + private static final Pattern PATTERN_WITHOUT_EXCAHNGE = tableFinish().with(source().matching( delete().with(source().matching( tableScan().capturedAs(TABLE_SCAN))))); - private final Metadata metadata; + private static final Pattern PATTERN_WITH_EXCAHNGE = + tableFinish().with(source().matching( + exchange().with(source().matching( + delete().with(source().matching( + tableScan().capturedAs(TABLE_SCAN))))))); - public PushDeleteIntoConnector(Metadata metadata) + private final Metadata metadata; + private final boolean withExchange; + + public PushDeleteIntoConnector(Metadata metadata, boolean withExchange) { this.metadata = requireNonNull(metadata, "metadata is null"); + this.withExchange = withExchange; } @Override public Pattern getPattern() { - return PATTERN; + if (withExchange) { + return PATTERN_WITH_EXCAHNGE; + } + + return PATTERN_WITHOUT_EXCAHNGE; } @Override diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushLimitIntoTableScan.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushLimitIntoTableScan.java index 428333874..25c281792 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushLimitIntoTableScan.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushLimitIntoTableScan.java @@ -61,7 +61,8 @@ public class PushLimitIntoTableScan result.getHandle(), tableScan.getOutputSymbols(), tableScan.getAssignments(), - tableScan.getEnforcedConstraint()); + tableScan.getEnforcedConstraint(), + tableScan.getPredicate()); if (!result.isLimitGuaranteed()) { node = new LimitNode(limit.getId(), node, limit.getCount(), limit.isPartial()); diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushPredicateIntoTableScan.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushPredicateIntoTableScan.java index 57083fe02..525f9802b 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushPredicateIntoTableScan.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushPredicateIntoTableScan.java @@ -57,7 +57,6 @@ import java.util.stream.Collectors; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.intersection; -import static io.prestosql.SystemSessionProperties.isPushTableWriteThroughUnion; import static io.prestosql.matching.Capture.newCapture; import static io.prestosql.metadata.TableLayoutResult.computeEnforced; import static io.prestosql.sql.ExpressionUtils.combineConjuncts; @@ -170,7 +169,7 @@ public class PushPredicateIntoTableScan AtomicInteger i = new AtomicInteger(0); additionalPredicates.stream().forEach(er -> { - log.warn("[%d]- Enforced [%s]\n\tRemaining [%s]", i.getAndIncrement(), + log.debug("[%d]- Enforced [%s]\n\tRemaining [%s]", i.getAndIncrement(), er.getTupleDomain(), er.getRemainingExpression()); }); @@ -184,10 +183,10 @@ public class PushPredicateIntoTableScan Map assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); - Boolean isPushDownEnabled = isPushTableWriteThroughUnion(session); - Constraint constraint; - List additionalConstraints = orDomains.stream().map(d -> new Constraint(d)) + List additionalConstraints = orDomains.stream() + .filter(d -> !d.isAll() && !d.isNone()) + .map(d -> new Constraint(d)) .collect(Collectors.toList()); if (pruneWithPredicateExpression) { @@ -220,7 +219,6 @@ public class PushPredicateIntoTableScan return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of())); } - constraint.setPushDownEnabled(isPushDownEnabled); Optional> result = metadata.applyFilter(session, node.getTable(), constraint, additionalConstraints); if (!result.isPresent()) { @@ -257,7 +255,8 @@ public class PushPredicateIntoTableScan newTable, node.getOutputSymbols(), node.getAssignments(), - computeEnforced(newDomain, remainingFilter)); + computeEnforced(newDomain, remainingFilter), + Optional.of(deterministicPredicate)); // The order of the arguments to combineConjuncts matters: // * Unenforced constraints go first because they can only be simple column references, diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushSampleIntoTableScan.java b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushSampleIntoTableScan.java index 4cf7113fc..b86ac9021 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushSampleIntoTableScan.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/iterative/rule/PushSampleIntoTableScan.java @@ -61,7 +61,8 @@ public class PushSampleIntoTableScan result, tableScan.getOutputSymbols(), tableScan.getAssignments(), - tableScan.getEnforcedConstraint()))) + tableScan.getEnforcedConstraint(), + tableScan.getPredicate()))) .orElseGet(Result::empty); } diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java index 91cc7356e..ca159162c 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/BeginTableWrite.java @@ -236,7 +236,8 @@ public class BeginTableWrite handle, scan.getOutputSymbols(), scan.getAssignments(), - scan.getEnforcedConstraint()); + scan.getEnforcedConstraint(), + scan.getPredicate()); } if (node instanceof FilterNode) { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneUnreferencedOutputs.java index 4404f64d0..7dde33126 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -442,7 +442,8 @@ public class PruneUnreferencedOutputs node.getTable(), newOutputs, newAssignments, - node.getEnforcedConstraint()); + node.getEnforcedConstraint(), + node.getPredicate()); } @Override diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/plan/TableScanNode.java b/presto-main/src/main/java/io/prestosql/sql/planner/plan/TableScanNode.java index 8742ecfbf..b8f5bce5e 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/plan/TableScanNode.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/plan/TableScanNode.java @@ -21,11 +21,13 @@ import io.prestosql.metadata.TableHandle; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.predicate.TupleDomain; import io.prestosql.sql.planner.Symbol; +import io.prestosql.sql.tree.Expression; import javax.annotation.concurrent.Immutable; import java.util.List; import java.util.Map; +import java.util.Optional; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; @@ -41,6 +43,7 @@ public class TableScanNode private final Map assignments; // symbol -> column private final TupleDomain enforcedConstraint; + private final Optional predicate; // We need this factory method to disambiguate with the constructor used for deserializing // from a json object. The deserializer sets some fields which are never transported @@ -51,7 +54,7 @@ public class TableScanNode List outputs, Map assignments) { - return new TableScanNode(id, table, outputs, assignments, TupleDomain.all()); + return new TableScanNode(id, table, outputs, assignments, TupleDomain.all(), Optional.empty()); } @JsonCreator @@ -59,7 +62,8 @@ public class TableScanNode @JsonProperty("id") PlanNodeId id, @JsonProperty("table") TableHandle table, @JsonProperty("outputSymbols") List outputs, - @JsonProperty("assignments") Map assignments) + @JsonProperty("assignments") Map assignments, + @JsonProperty("predicate") Optional predicate) { // This constructor is for JSON deserialization only. Do not use. super(id); @@ -68,6 +72,7 @@ public class TableScanNode this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs"); this.enforcedConstraint = null; + this.predicate = predicate; } public TableScanNode( @@ -75,7 +80,8 @@ public class TableScanNode TableHandle table, List outputs, Map assignments, - TupleDomain enforcedConstraint) + TupleDomain enforcedConstraint, + Optional predicate) { super(id); this.table = requireNonNull(table, "table is null"); @@ -83,6 +89,7 @@ public class TableScanNode this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); checkArgument(assignments.keySet().containsAll(outputs), "assignments does not cover all of outputs"); this.enforcedConstraint = requireNonNull(enforcedConstraint, "enforcedConstraint is null"); + this.predicate = requireNonNull(predicate, "predicate expression cannot be empty"); } @JsonProperty("table") @@ -119,6 +126,12 @@ public class TableScanNode return enforcedConstraint; } + @JsonProperty("predicate") + public Optional getPredicate() + { + return predicate; + } + @Override public List getSources() { diff --git a/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java b/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java index 6897f43a3..bc3bcdc8d 100644 --- a/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java +++ b/presto-main/src/main/java/io/prestosql/sql/planner/planprinter/PlanPrinter.java @@ -759,6 +759,12 @@ public class PlanPrinter formatString += "grouped = %s, "; arguments.add(stageExecutionStrategy.get().isScanGroupedExecution(scanNode.get().getId())); } + + String pushdownPredicates = printTablePushDownFilters(scanNode.get()); + if (pushdownPredicates.length() > 0) { + formatString += "pushdownFilters = %s, "; + arguments.add(pushdownPredicates); + } } if (filterNode.isPresent()) { @@ -830,6 +836,38 @@ public class PlanPrinter .collect(Collectors.joining(", ", "{", "}")); } + private String printTablePushDownFilters(TableScanNode node) + { + StringBuilder str = new StringBuilder(); + TupleDomain predicate = tableInfoSupplier.apply(node).getPredicate(); + if (!predicate.isNone() && !predicate.isAll()) { + if (predicate.isAll() && (!node.getEnforcedConstraint().isAll() || !node.getEnforcedConstraint().isNone())) { + predicate = node.getEnforcedConstraint(); + } + + if (!predicate.isNone() && !predicate.isAll()) { + str.append("[ "); + str.append(predicate.getDomains().get() + .entrySet().stream() + .map(filter -> filter.getKey() + " <- " + formatDomain(filter.getValue().simplify())) + .collect(Collectors.joining(" AND ", "{", "}"))); + str.append(" ]"); + if (node.getTable().getConnectorHandle().hasAdditionalFiltersPushdown()) { + str.append("[ "); + str.append(" AND "); + str.append(" ]"); + } + } + + if (node.getTable().getConnectorHandle().hasAdditionalFiltersPushdown()) { + str.append(node.getTable().getConnectorHandle() + .getAdditionalFilterConditions((domain) -> formatDomain(domain))); + } + } + + return str.toString(); + } + private void printTableScanInfo(NodeRepresentation nodeOutput, TableScanNode node) { TupleDomain predicate = tableInfoSupplier.apply(node).getPredicate(); diff --git a/presto-main/src/main/java/io/prestosql/utils/PredicateExtractor.java b/presto-main/src/main/java/io/prestosql/utils/PredicateExtractor.java index c149594e1..6df9fbd43 100644 --- a/presto-main/src/main/java/io/prestosql/utils/PredicateExtractor.java +++ b/presto-main/src/main/java/io/prestosql/utils/PredicateExtractor.java @@ -44,6 +44,7 @@ import java.sql.Timestamp; import java.time.LocalDate; import java.util.HashMap; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -68,52 +69,87 @@ public class PredicateExtractor { } - private static Optional getFilterNode(SqlStageExecution stage) + private static List getFilterNode(SqlStageExecution stage) { PlanFragment fragment = stage.getFragment(); PlanNode root = fragment.getRoot(); + List result = new LinkedList<>(); Queue queue = new LinkedList<>(); queue.add(root); while (!queue.isEmpty()) { PlanNode node = queue.poll(); - if (node instanceof FilterNode) { - return Optional.of((FilterNode) node); + if (node instanceof FilterNode + || node instanceof TableScanNode) { + result.add(node); } queue.addAll(node.getSources()); } - return Optional.empty(); + return result; } public static boolean isSplitFilterApplicable(SqlStageExecution stage) { - Optional filterNodeOptional = getFilterNode(stage); + List filterNodeOptional = getFilterNode(stage); - if (!filterNodeOptional.isPresent()) { + if (filterNodeOptional.isEmpty()) { return false; } - FilterNode filterNode = filterNodeOptional.get(); + PlanNode node = filterNodeOptional.get(0); - PlanNode sourceNode = filterNode.getSource(); - if (!(sourceNode instanceof TableScanNode)) { - return false; + if (node instanceof FilterNode) { + FilterNode filterNode = (FilterNode) node; + PlanNode sourceNode = filterNode.getSource(); + if (!(sourceNode instanceof TableScanNode)) { + return false; + } + + //if a catalog name starts with a $, it's not an normal query, could be something like show tables; + TableHandle table = ((TableScanNode) sourceNode).getTable(); + String catalogName = table.getCatalogName().getCatalogName(); + if (catalogName.startsWith("$")) { + return false; + } + + /* (!(table.getConnectorHandle().isFilterSupported() + * && (isSupportedExpression(filterNode.getPredicate()) + * || (((TableScanNode) sourceNode).getPredicate().isPresent() + * && isSupportedExpression(((TableScanNode) sourceNode).getPredicate().get()))))) + */ + if (!table.getConnectorHandle().isFilterSupported()) { + return false; + } + + if (!isSupportedExpression(filterNode.getPredicate()) + && (!((TableScanNode) sourceNode).getPredicate().isPresent() + || !isSupportedExpression(((TableScanNode) sourceNode).getPredicate().get()))) { + return false; + } } - //if a catalog name starts with a $, it's not an normal query, could be something like show tables; - TableHandle table = ((TableScanNode) sourceNode).getTable(); - String catalogName = table.getCatalogName().getCatalogName(); - if (catalogName.startsWith("$")) { - return false; + if (node instanceof TableScanNode) { + TableScanNode tableScanNode = (TableScanNode) node; + //if a catalog name starts with a $, it's not an normal query, could be something like show tables; + TableHandle table = tableScanNode.getTable(); + String catalogName = table.getCatalogName().getCatalogName(); + if (catalogName.startsWith("$")) { + return false; + } + + if (!table.getConnectorHandle().isFilterSupported()) { + return false; + } + + if (!tableScanNode.getPredicate().isPresent() + || !isSupportedExpression(tableScanNode.getPredicate().get())) { + return false; + } } - boolean supported = table.getConnectorHandle().isFilterSupported(); - if (!supported || !isSupportedExpression(filterNode.getPredicate())) { - return false; - } return true; } @@ -142,6 +178,7 @@ public class PredicateExtractor if (predicate instanceof InPredicate) { return true; } + if (predicate instanceof NotExpression) { return true; } @@ -158,31 +195,53 @@ public class PredicateExtractor */ public static Tuple, Map> getExpression(SqlStageExecution stage) { - Optional filterNodeOptional = getFilterNode(stage); + List filterNodeOptional = getFilterNode(stage); - if (!filterNodeOptional.isPresent()) { + if (filterNodeOptional.size() == 0) { return new Tuple<>(Optional.empty(), new HashMap<>()); } - FilterNode filterNode = filterNodeOptional.get(); - if (filterNode.getSource() instanceof TableScanNode) { - TableScanNode tableScanNode = (TableScanNode) filterNode.getSource(); - Expression predicate = filterNode.getPredicate(); - return new Tuple<>(Optional.of(predicate), tableScanNode.getAssignments()); + if (filterNodeOptional.get(0) instanceof FilterNode) { + FilterNode filterNode = (FilterNode) filterNodeOptional.get(0); + if (filterNode.getSource() instanceof TableScanNode) { + TableScanNode tableScanNode = (TableScanNode) filterNode.getSource(); + if (tableScanNode.getPredicate().isPresent()) { + return new Tuple<>(tableScanNode.getPredicate(), tableScanNode.getAssignments()); + } + + return new Tuple<>(Optional.of(filterNode.getPredicate()), tableScanNode.getAssignments()); + } + + return new Tuple<>(Optional.empty(), new HashMap<>()); } + + if (filterNodeOptional.get(0) instanceof TableScanNode) { + TableScanNode tableScanNode = (TableScanNode) filterNodeOptional.get(0); + if (tableScanNode.getPredicate().isPresent()) { + return new Tuple<>(tableScanNode.getPredicate(), tableScanNode.getAssignments()); + } + } + return new Tuple<>(Optional.empty(), new HashMap<>()); } public static Optional getFullyQualifiedName(SqlStageExecution stage) { - Optional filterNodeOptional = getFilterNode(stage); + List filterNodeOptional = getFilterNode(stage); - if (!filterNodeOptional.isPresent()) { + if (filterNodeOptional.size() == 0) { return Optional.empty(); } - FilterNode filterNode = filterNodeOptional.get(); - TableScanNode tableScanNode = (TableScanNode) filterNode.getSource(); + TableScanNode tableScanNode; + if (filterNodeOptional.get(0) instanceof FilterNode) { + FilterNode filterNode = (FilterNode) filterNodeOptional.get(0); + tableScanNode = (TableScanNode) filterNode.getSource(); + } + else { + tableScanNode = (TableScanNode) filterNodeOptional.get(0); + } + String fullQualifiedTableName = tableScanNode.getTable().getFullyQualifiedName(); return Optional.of(fullQualifiedTableName); diff --git a/presto-main/src/main/java/io/prestosql/vacuum/AutoVacuumScanner.java b/presto-main/src/main/java/io/prestosql/vacuum/AutoVacuumScanner.java index fb9dad596..a9bd88a2d 100644 --- a/presto-main/src/main/java/io/prestosql/vacuum/AutoVacuumScanner.java +++ b/presto-main/src/main/java/io/prestosql/vacuum/AutoVacuumScanner.java @@ -271,7 +271,6 @@ public class AutoVacuumScanner return connectorMetadata.getTablesForVacuum(); } catch (UnsupportedOperationException e) { - log.error("Catalog[%s] does not support auto vacuum scan", catalog.getCatalogName()); return null; } catch (Exception e) { diff --git a/presto-main/src/test/java/io/prestosql/cost/TestCostCalculator.java b/presto-main/src/test/java/io/prestosql/cost/TestCostCalculator.java index c2cf7b2cf..c6c275336 100644 --- a/presto-main/src/test/java/io/prestosql/cost/TestCostCalculator.java +++ b/presto-main/src/test/java/io/prestosql/cost/TestCostCalculator.java @@ -797,7 +797,8 @@ public class TestCostCalculator new TableHandle(new CatalogName("tpch"), tableHandle, INSTANCE, Optional.of(new TpchTableLayoutHandle(tableHandle, TupleDomain.all()))), symbolsList, assignments.build(), - TupleDomain.all()); + TupleDomain.all(), + Optional.empty()); } private PlanNode project(String id, PlanNode source, String symbol, Expression expression) diff --git a/presto-main/src/test/java/io/prestosql/sql/planner/TestEffectivePredicateExtractor.java b/presto-main/src/test/java/io/prestosql/sql/planner/TestEffectivePredicateExtractor.java index c119a0b5d..197713f85 100644 --- a/presto-main/src/test/java/io/prestosql/sql/planner/TestEffectivePredicateExtractor.java +++ b/presto-main/src/test/java/io/prestosql/sql/planner/TestEffectivePredicateExtractor.java @@ -409,7 +409,8 @@ public class TestEffectivePredicateExtractor makeTableHandle(TupleDomain.none()), ImmutableList.copyOf(assignments.keySet()), assignments, - TupleDomain.none()); + TupleDomain.none(), + Optional.empty()); effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer); assertEquals(effectivePredicate, FALSE_LITERAL); TupleDomain predicate = TupleDomain.withColumnDomains(ImmutableMap.of(scanAssignments.get(A), Domain.singleValue(BIGINT, 1L))); @@ -418,7 +419,8 @@ public class TestEffectivePredicateExtractor makeTableHandle(predicate), ImmutableList.copyOf(assignments.keySet()), assignments, - predicate); + predicate, + Optional.empty()); effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer); assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(1L), AE))); predicate = TupleDomain.withColumnDomains(ImmutableMap.of( @@ -429,7 +431,8 @@ public class TestEffectivePredicateExtractor makeTableHandle(TupleDomain.withColumnDomains(ImmutableMap.of(scanAssignments.get(A), Domain.singleValue(BIGINT, 1L)))), ImmutableList.copyOf(assignments.keySet()), assignments, - predicate); + predicate, + Optional.empty()); effectivePredicate = effectivePredicateExtractorWithoutTableProperties.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer); assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(2L), BE), equals(bigintLiteral(1L), AE))); @@ -438,7 +441,8 @@ public class TestEffectivePredicateExtractor makeTableHandle(predicate), ImmutableList.copyOf(assignments.keySet()), assignments, - TupleDomain.all()); + TupleDomain.all(), + Optional.empty()); effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer); assertEquals(effectivePredicate, and(equals(AE, bigintLiteral(1)), equals(BE, bigintLiteral(2)))); @@ -449,7 +453,8 @@ public class TestEffectivePredicateExtractor assignments, TupleDomain.withColumnDomains(ImmutableMap.of( scanAssignments.get(A), Domain.multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L)), - scanAssignments.get(B), Domain.multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L))))); + scanAssignments.get(B), Domain.multipleValues(BIGINT, ImmutableList.of(1L, 2L, 3L)))), + Optional.empty()); effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer); assertEquals(normalizeConjuncts(effectivePredicate), normalizeConjuncts(equals(bigintLiteral(2L), BE), equals(bigintLiteral(1L), AE))); @@ -458,7 +463,8 @@ public class TestEffectivePredicateExtractor makeTableHandle(TupleDomain.all()), ImmutableList.copyOf(assignments.keySet()), assignments, - TupleDomain.all()); + TupleDomain.all(), + Optional.empty()); effectivePredicate = effectivePredicateExtractor.extract(SESSION, node, TypeProvider.empty(), typeAnalyzer); assertEquals(effectivePredicate, BooleanLiteral.TRUE_LITERAL); } @@ -929,7 +935,8 @@ public class TestEffectivePredicateExtractor makeTableHandle(TupleDomain.all()), ImmutableList.copyOf(scanAssignments.keySet()), scanAssignments, - TupleDomain.all()); + TupleDomain.all(), + Optional.empty()); } private static PlanNodeId newId() diff --git a/presto-main/src/test/java/io/prestosql/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/io/prestosql/sql/planner/TestLogicalPlanner.java index e95698193..be07e546b 100644 --- a/presto-main/src/test/java/io/prestosql/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/io/prestosql/sql/planner/TestLogicalPlanner.java @@ -736,10 +736,11 @@ public class TestLogicalPlanner assertPlan( "SELECT orderkey FROM orders WHERE orderstatus='F'", output( - constrainedTableScanWithTableLayout( + project( + constrainedTableScanWithTableLayout( "orders", ImmutableMap.of("orderstatus", singleValue(createVarcharType(1), utf8Slice("F"))), - ImmutableMap.of("orderkey", "orderkey")))); + ImmutableMap.of("orderkey", "orderkey"))))); } @Test diff --git a/presto-main/src/test/java/io/prestosql/sql/planner/TestPredicatePushdown.java b/presto-main/src/test/java/io/prestosql/sql/planner/TestPredicatePushdown.java index 2ec39fafb..a4efda902 100644 --- a/presto-main/src/test/java/io/prestosql/sql/planner/TestPredicatePushdown.java +++ b/presto-main/src/test/java/io/prestosql/sql/planner/TestPredicatePushdown.java @@ -477,7 +477,7 @@ public class TestPredicatePushdown "orders", ImmutableMap.of("ORDERSTATUS", "orderstatus"))), anyTree( - filter("CAST(NAME AS varchar(1)) IN ('F', 'O')", + filter("CAST(NAME AS varchar(1)) BETWEEN 'A' AND 'O'", tableScan( "nation", ImmutableMap.of("NAME", "name"))))))); diff --git a/presto-main/src/test/java/io/prestosql/sql/planner/TestTypeValidator.java b/presto-main/src/test/java/io/prestosql/sql/planner/TestTypeValidator.java index 8ee1fa996..83dc0cad7 100644 --- a/presto-main/src/test/java/io/prestosql/sql/planner/TestTypeValidator.java +++ b/presto-main/src/test/java/io/prestosql/sql/planner/TestTypeValidator.java @@ -97,7 +97,8 @@ public class TestTypeValidator TEST_TABLE_HANDLE, ImmutableList.copyOf(assignments.keySet()), assignments, - TupleDomain.all()); + TupleDomain.all(), + Optional.empty()); } @Test diff --git a/presto-main/src/test/java/io/prestosql/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/io/prestosql/sql/planner/iterative/rule/test/PlanBuilder.java index 14e65766b..43940ee95 100644 --- a/presto-main/src/test/java/io/prestosql/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/io/prestosql/sql/planner/iterative/rule/test/PlanBuilder.java @@ -456,7 +456,8 @@ public class PlanBuilder tableHandle, symbols, assignments, - enforcedConstraint); + enforcedConstraint, + Optional.empty()); } public TableFinishNode tableDelete(SchemaTableName schemaTableName, PlanNode deleteSource, Symbol deleteRowId) diff --git a/presto-main/src/test/java/io/prestosql/utils/TestUtil.java b/presto-main/src/test/java/io/prestosql/utils/TestUtil.java index 44fa1ff5b..5a5818d23 100644 --- a/presto-main/src/test/java/io/prestosql/utils/TestUtil.java +++ b/presto-main/src/test/java/io/prestosql/utils/TestUtil.java @@ -120,7 +120,8 @@ public class TestUtil makeTableHandle(TupleDomain.none()), ImmutableList.copyOf(assignments.keySet()), assignments, - TupleDomain.none()); + TupleDomain.none(), + Optional.empty()); PlanBuilder planBuilder = new PlanBuilder(new PlanNodeIdAllocator(), dummyMetadata()); FilterNode filterNode = planBuilder.filter(expr, tableScanNode); diff --git a/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java b/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java index 2f638662c..50b674ce7 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/AbstractOrcRecordReader.java @@ -36,7 +36,8 @@ import io.prestosql.orc.metadata.statistics.ColumnStatistics; import io.prestosql.orc.metadata.statistics.StripeStatistics; import io.prestosql.orc.reader.AbstractColumnReader; import io.prestosql.orc.reader.CachingColumnReader; -import io.prestosql.orc.reader.SelectiveCachingColumnReader; +import io.prestosql.orc.reader.DataCachingSelectiveColumnReader; +import io.prestosql.orc.reader.ResultCachingSelectiveColumnReader; import io.prestosql.orc.stream.InputStreamSources; import io.prestosql.orc.stream.StreamSourceMeta; import io.prestosql.spi.Page; @@ -153,7 +154,8 @@ abstract class AbstractOrcRecordReader Optional> indexes, Map domains, OrcCacheStore orcCacheStore, - OrcCacheProperties orcCacheProperties) + OrcCacheProperties orcCacheProperties, + Map> orDomains) throws OrcCorruptionException { requireNonNull(readColumns, "readColumns is null"); @@ -183,6 +185,7 @@ abstract class AbstractOrcRecordReader // it is possible that old versions of orc use 0 to mean there are no row groups checkArgument(rowsInRowGroup > 0, "rowsInRowGroup must be greater than zero"); + checkArgument(orDomains != null, "orDomain map cannot be null"); // sort stripes by file position List stripeInfos = new ArrayList<>(); @@ -225,14 +228,12 @@ abstract class AbstractOrcRecordReader for (int i = 0; i < stripeInfos.size(); i++) { StripeInfo info = stripeInfos.get(i); StripeInformation stripe = info.getStripe(); - if (splitContainsStripe(splitOffset, splitLength, stripe) && isStripeIncluded(stripe, info.getStats(), predicate)) { + if (splitContainsStripe(splitOffset, splitLength, stripe) + && isStripeIncluded(stripe, info.getStats(), predicate) + && !filterStripeUsingIndex(stripe, stripeOffsetToIndex, domains, orDomains)) { stripes.add(stripe); stripeFilePositions.add(fileRowCount); totalRowCount += stripe.getNumberOfRows(); - - if (!stripeOffsetToIndex.isEmpty()) { - stripeIndexes.put(stripe, stripeOffsetToIndex.get(stripe.getOffset())); - } } fileRowCount += stripe.getNumberOfRows(); } @@ -241,37 +242,6 @@ abstract class AbstractOrcRecordReader this.stripes = stripes.build(); this.stripeFilePositions = stripeFilePositions.build(); - // now that we know which stripes will be read, apply indexes on them if applicable - // i.e. if an index exists for the pushed down predicates - // once the indexes are applied, for each stripe we will have the rows inside - // the stripe that matched the predicates - stripeIndexes.entrySet().stream().forEach(stripeIndex -> { - Map indexDomainMap = new HashMap<>(); - - for (Map.Entry domainEntry : domains.entrySet()) { - String columnName = domainEntry.getKey(); - Domain columnDomain = domainEntry.getValue(); - - // if the index exists, there should only be one index for this column within this stripe - List indexMetadata = stripeIndex.getValue().stream().filter(p -> p.getColumn().equalsIgnoreCase(columnName)).collect(Collectors.toList()); - if (indexMetadata.isEmpty() || indexMetadata.size() > 1) { - continue; - } - - Index index = indexMetadata.get(0).getIndex(); - indexDomainMap.put(index, columnDomain); - } - - if (!indexDomainMap.isEmpty()) { - Iterator thisStripeMatchingRows = indexDomainMap.entrySet().iterator().next().getKey().getMatches(indexDomainMap); - - if (thisStripeMatchingRows != null) { - PeekingIterator peekingIterator = Iterators.peekingIterator(thisStripeMatchingRows); - stripeMatchingRows.put(stripeIndex.getKey(), peekingIterator); - } - } - }); - orcDataSource = wrapWithCacheIfTinyStripes(orcDataSource, this.stripes, maxMergeDistance, tinyStripeThreshold); this.orcDataSource = orcDataSource; this.splitLength = splitLength; @@ -314,6 +284,69 @@ abstract class AbstractOrcRecordReader } } + private boolean filterStripeUsingIndex(StripeInformation stripe, Map> stripeOffsetToIndex, + Map and, Map> or) + { + if (stripeOffsetToIndex.isEmpty()) { + return false; + } + + List stripeIndex = stripeOffsetToIndex.get(Long.valueOf(stripe.getOffset())); + Map andDomainMap = new HashMap<>(); + Map orDomainMap = new HashMap<>(); + + for (Map.Entry domainEntry : and.entrySet()) { + String columnName = domainEntry.getKey(); + Domain columnDomain = domainEntry.getValue(); + + // if the index exists, there should only be one index for this column within this stripe + List indexMetadata = stripeIndex.stream().filter(p -> p.getColumn().equalsIgnoreCase(columnName)).collect(Collectors.toList()); + if (indexMetadata.isEmpty() || indexMetadata.size() > 1) { + continue; + } + + Index index = indexMetadata.get(0).getIndex(); + andDomainMap.put(index, columnDomain); + } + + for (Map.Entry> domainEntry : or.entrySet()) { + String columnName = domainEntry.getKey(); + List columnDomain = domainEntry.getValue(); + + // if the index exists, there should only be one index for this column within this stripe + List indexMetadata = stripeIndex.stream().filter(p -> p.getColumn().equalsIgnoreCase(columnName)).collect(Collectors.toList()); + if (indexMetadata.isEmpty() || indexMetadata.size() > 1) { + continue; + } + + Index index = indexMetadata.get(0).getIndex(); + orDomainMap.put(index, columnDomain.get(0)); + } + + if (!andDomainMap.isEmpty()) { + Iterator thisStripeMatchingRows = ((andDomainMap.entrySet().iterator().next()).getKey()).getMatches(andDomainMap); + if (thisStripeMatchingRows.hasNext()) { + PeekingIterator peekingIterator = Iterators.peekingIterator(thisStripeMatchingRows); + if (peekingIterator.peek() != null) { + this.stripeMatchingRows.put(stripe, peekingIterator); + } + return false; + } + return true; + } + if (!orDomainMap.isEmpty()) { + for (Map.Entry indexDomainEntry : orDomainMap.entrySet()) { + Iterator thisStripeMatchingRows = (indexDomainEntry.getKey()).getMatches(indexDomainEntry.getValue()); + if (thisStripeMatchingRows.hasNext()) { + /* any one matched; then include the stripe */ + return false; + } + } + return true; + } + return false; + } + private static OptionalInt getFixedWidthRowSize(List columnTypes) { int totalFixedWidth = 0; @@ -642,7 +675,9 @@ abstract class AbstractOrcRecordReader InputStreamSources rowGroupStreamSources = currentRowGroup.getStreamSources(); for (AbstractColumnReader columnReader : columnReaders) { if (columnReader != null) { - if (columnReader instanceof CachingColumnReader || columnReader instanceof SelectiveCachingColumnReader) { + if (columnReader instanceof CachingColumnReader + || columnReader instanceof ResultCachingSelectiveColumnReader + || columnReader instanceof DataCachingSelectiveColumnReader) { StreamSourceMeta streamSourceMeta = new StreamSourceMeta(); streamSourceMeta.setDataSourceId(orcDataSource.getId()); streamSourceMeta.setStripeOffset(stripes.get(currentStripe).getOffset()); diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java b/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java index bd34e972b..bcef1da3d 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/OrcReader.java @@ -28,6 +28,7 @@ import io.prestosql.orc.metadata.OrcType; import io.prestosql.orc.metadata.OrcType.OrcTypeKind; import io.prestosql.orc.metadata.PostScript.HiveWriterVersion; import io.prestosql.spi.Page; +import io.prestosql.spi.block.Block; import io.prestosql.spi.heuristicindex.IndexMetadata; import io.prestosql.spi.predicate.Domain; import io.prestosql.spi.type.Type; @@ -315,7 +316,9 @@ public class OrcReader OrcCacheProperties orcCacheProperties, Optional writeValidation, Map> additionalFilters, - List positions) throws OrcCorruptionException + List positions, boolean useDataCache, + Map> coercer, + Map> orDomains) throws OrcCorruptionException { return new OrcSelectiveRecordReader( outputColumns, @@ -352,7 +355,10 @@ public class OrcReader orcCacheStore, orcCacheProperties, additionalFilters, - positions); + positions, + useDataCache, + coercer, + orDomains); } public static OrcDataSource wrapWithCacheIfTiny(OrcDataSource dataSource, DataSize maxCacheSize) diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java b/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java index 070267bc8..522e06bbd 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/OrcRecordReader.java @@ -115,7 +115,8 @@ public class OrcRecordReader indexes, domains, orcCacheStore, - orcCacheProperties); + orcCacheProperties, + ImmutableMap.of()); setColumnReadersParam(createColumnReaders(readColumns, readTypes, systemMemoryUsage.newAggregatedMemoryContext(), diff --git a/presto-orc/src/main/java/io/prestosql/orc/OrcSelectiveRecordReader.java b/presto-orc/src/main/java/io/prestosql/orc/OrcSelectiveRecordReader.java index 0322a0d88..6f5ce8947 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/OrcSelectiveRecordReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/OrcSelectiveRecordReader.java @@ -13,6 +13,7 @@ */ package io.prestosql.orc; +import com.google.common.collect.PeekingIterator; import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.airlift.units.DataSize; @@ -25,6 +26,8 @@ import io.prestosql.orc.metadata.PostScript; import io.prestosql.orc.metadata.StripeInformation; import io.prestosql.orc.metadata.statistics.ColumnStatistics; import io.prestosql.orc.metadata.statistics.StripeStatistics; +import io.prestosql.orc.reader.ColumnReader; +import io.prestosql.orc.reader.ColumnReaders; import io.prestosql.orc.reader.SelectiveColumnReader; import io.prestosql.orc.reader.SelectiveColumnReaders; import io.prestosql.spi.Page; @@ -39,14 +42,18 @@ import org.openjdk.jol.info.ClassLayout; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.stream.IntStream; +import static com.google.common.base.Preconditions.checkArgument; import static io.prestosql.orc.reader.SelectiveColumnReaders.createColumnReader; +import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; public class OrcSelectiveRecordReader @@ -57,17 +64,17 @@ public class OrcSelectiveRecordReader private final List excludePositions; private final List columnReaderOrder; + private final Map filters; private int[] positions; List outputColumns; Map includedColumns; - private int finalPositionCount; - private int[] finalPositions; private final Map constantValues; Set colReaderWithFilter; Set colReaderWithORFilter; Set colReaderWithoutFilter; Map> additionalFilters; + Map> coercers; public OrcSelectiveRecordReader( List outputColumns, @@ -104,7 +111,10 @@ public class OrcSelectiveRecordReader OrcCacheStore orcCacheStore, OrcCacheProperties orcCacheProperties, Map> additionalFilters, - List positions) + List positions, + boolean useDataCache, + Map> coercers, + Map> orDomains) throws OrcCorruptionException { super(fileReadColumns, @@ -134,7 +144,8 @@ public class OrcSelectiveRecordReader indexes, domains, orcCacheStore, - orcCacheProperties); + orcCacheProperties, + orDomains); int fieldCount = orcTypes.get(OrcColumnId.ROOT_COLUMN).getFieldCount(); this.columnReaderOrder = new ArrayList<>(fieldCount); @@ -143,15 +154,21 @@ public class OrcSelectiveRecordReader this.includedColumns = includedColumns; this.excludePositions = positions; + this.filters = filters; this.additionalFilters = additionalFilters; this.constantValues = requireNonNull(constantValues, "constantValues is null"); + this.coercers = requireNonNull(coercers, "coercers is null"); + + for (Map.Entry> entry : coercers.entrySet()) { + checkArgument(!filters.containsKey(entry.getKey()), "Coercions for columns with range filters are not yet supported"); + } setColumnReadersParam(createColumnReaders(fileColumns, systemMemoryUsage.newAggregatedMemoryContext(), new OrcBlockFactory(exceptionTransform, true), orcCacheStore, orcCacheProperties, predicate, filters, hiveStorageTimeZone, - outputColumns, includedColumns, orcTypes)); + outputColumns, includedColumns, orcTypes, useDataCache)); } public Page getNextPage() @@ -162,6 +179,7 @@ public class OrcSelectiveRecordReader return null; } + matchingRowsInBatchArray = null; initializePositions(batchSize); int[] positionsToRead = this.positions; @@ -172,7 +190,7 @@ public class OrcSelectiveRecordReader if (positionCount != 0) { for (Integer columnIdx : colReaderWithFilter) { if (columnReaders[columnIdx] != null) { - positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount); + positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount, filters.get(columnIdx)); if (positionCount == 0) { break; } @@ -196,13 +214,13 @@ public class OrcSelectiveRecordReader int[] newPositions = positionsToRead.clone(); positionCount = updateExcludePositions(positionsToRead, positionCount, accumulator, newPositions); - positionsToRead = newPositions; + positionsToRead = Arrays.copyOf(newPositions, positionCount); } if (positionCount != 0) { for (Integer columnIdx : colReaderWithoutFilter) { if (columnReaders[columnIdx] != null) { - positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount); + positionCount = columnReaders[columnIdx].read(getNextRowInGroup(), positionsToRead, positionCount, null); if (positionCount == 0) { break; } @@ -233,6 +251,9 @@ public class OrcSelectiveRecordReader else { Block block = getColumnReaders()[columnIndex].getBlock(positionsToRead, positionCount); updateMaxCombinedBytesPerRow(columnIndex, block); + if (coercers.containsKey(i)) { + block = coercers.get(i).apply(block); + } blocks[i] = block; } } @@ -246,7 +267,38 @@ public class OrcSelectiveRecordReader private void initializePositions(int batchSize) { + // currentPosition to currentBatchSize + StripeInformation stripe = stripes.get(currentStripe); + + if (matchingRowsInBatchArray == null && stripeMatchingRows.containsKey(stripe)) { + long currentPositionInStripe = currentPosition - currentStripePosition; + + PeekingIterator matchingRows = stripeMatchingRows.get(stripe); + List matchingRowsInBlock = new ArrayList<>(); + + while (matchingRows.hasNext()) { + Integer row = matchingRows.peek(); + if (row >= currentPositionInStripe && row < currentPositionInStripe + batchSize) { + matchingRowsInBlock.add(toIntExact(Long.valueOf(row) - currentPositionInStripe)); + matchingRows.next(); + } + else if (row >= currentPositionInStripe + currentBatchSize) { + break; + } + } + + matchingRowsInBatchArray = new int[matchingRowsInBlock.size()]; + IntStream.range(0, matchingRowsInBlock.size()).forEach( + i -> matchingRowsInBatchArray[i] = matchingRowsInBlock.get(i)); + log.debug("Find matching rows from stripe. Matching row count for the block = %d", matchingRowsInBatchArray.length); + } + if (positions == null || positions.length < batchSize) { + if (matchingRowsInBatchArray != null) { + positions = matchingRowsInBatchArray; + return; + } + positions = new int[batchSize]; for (int i = 0; i < batchSize; i++) { positions[i] = i; @@ -284,7 +336,8 @@ public class OrcSelectiveRecordReader DateTimeZone hiveStorageTimeZone, List outputColumns, Map includedColumns, - ColumnMetadata orcTypes) + ColumnMetadata orcTypes, + boolean useDataCache) throws OrcCorruptionException { int fieldCount = orcTypes.get(OrcColumnId.ROOT_COLUMN).getFieldCount(); @@ -298,16 +351,28 @@ public class OrcSelectiveRecordReader int columnIndex = i; OrcColumn column = fileColumns.get(columnIndex); boolean outputRequired = outputColumns.contains(i); - SelectiveColumnReader columnReader = createColumnReader( - orcTypes.get(column.getColumnId()), - column, - Optional.ofNullable(filters.get(i)), - outputRequired ? Optional.of(includedColumns.get(i)) : Optional.empty(), - hiveStorageTimeZone, - systemMemoryContext); - if (orcCacheProperties.isRowDataCacheEnabled()) { - columnReader = SelectiveColumnReaders.wrapWithCachingStreamReader(columnReader, column, - predicate, orcCacheStore.getRowDataCache()); + SelectiveColumnReader columnReader = null; + + if (useDataCache && orcCacheProperties.isRowDataCacheEnabled()) { + ColumnReader cr = ColumnReaders.createColumnReader( + includedColumns.get(i), + column, + systemMemoryContext, + blockFactory.createNestedBlockFactory(block -> blockLoaded(columnIndex, block))); + columnReader = SelectiveColumnReaders.wrapWithDataCachingStreamReader(cr, column, orcCacheStore.getRowDataCache()); + } + else { + columnReader = createColumnReader( + orcTypes.get(column.getColumnId()), + column, + Optional.ofNullable(filters.get(i)), + outputRequired ? Optional.of(includedColumns.get(i)) : Optional.empty(), + hiveStorageTimeZone, + systemMemoryContext); + if (orcCacheProperties.isRowDataCacheEnabled()) { + columnReader = SelectiveColumnReaders.wrapWithResultCachingStreamReader(columnReader, column, + predicate, orcCacheStore.getRowDataCache()); + } } columnReaders[columnIndex] = columnReader; if (filters.get(i) != null) { diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractDecimalSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractDecimalSelectiveColumnReader.java index 530065e8c..046fb93bc 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractDecimalSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractDecimalSelectiveColumnReader.java @@ -46,8 +46,8 @@ import static io.prestosql.orc.metadata.Stream.StreamKind.SECONDARY; import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSource; import static java.util.Objects.requireNonNull; -public abstract class AbstractDecimalSelectiveColumnReader - implements SelectiveColumnReader +public abstract class AbstractDecimalSelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(AbstractDecimalSelectiveColumnReader.class).instanceSize(); @@ -135,11 +135,11 @@ public abstract class AbstractDecimalSelectiveColumnReader } @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { return readOr(offset, positions, positionCount, - (filter == null) ? null : ImmutableList.of(filter), + (this.filter == null) ? null : ImmutableList.of(this.filter), null); } @@ -231,7 +231,7 @@ public abstract class AbstractDecimalSelectiveColumnReader } @Override - public Block getBlock(int[] positions, int positionCount) + public Block getBlock(int[] positions, int positionCount) { checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); checkState(outputRequired, "This stream reader doesn't produce output"); @@ -264,9 +264,8 @@ public abstract class AbstractDecimalSelectiveColumnReader private void ensureValuesCapacity(int capacity, boolean nullAllowed) { - int valuesCapacity = valuesPerPosition * capacity; - if (values == null || values.length < valuesCapacity) { - values = new long[valuesCapacity]; + if (values == null || values.length < capacity) { + values = new long[valuesPerPosition * capacity]; } if (nullAllowed) { @@ -278,7 +277,7 @@ public abstract class AbstractDecimalSelectiveColumnReader abstract void copyValues(int[] positions, int positionsCount, long[] valuesCopy, boolean[] nullsCopy); - abstract Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values); + abstract Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values); abstract void compactValues(int[] positions, int positionCount, boolean compactNulls); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractLongSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractLongSelectiveColumnReader.java index 97ef8bff8..eee16cc7c 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractLongSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/AbstractLongSelectiveColumnReader.java @@ -35,8 +35,8 @@ import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; //TODO: Rajeev: To be combined with LongSelectiveColumnReader -abstract class AbstractLongSelectiveColumnReader - implements SelectiveColumnReader +abstract class AbstractLongSelectiveColumnReader + implements SelectiveColumnReader { protected final boolean outputRequired; @Nullable @@ -63,7 +63,7 @@ abstract class AbstractLongSelectiveColumnReader } @Override - public Block mergeBlocks(List blocks, int positionCount) + public Block mergeBlocks(List> blocks, int positionCount) { if (outputType == BIGINT) { LongArrayBlockBuilder blockBuilder = new LongArrayBlockBuilder(null, positionCount); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanColumnReader.java index 4b74abe51..b1b1904bf 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnMetadata; import io.prestosql.orc.stream.BooleanInputStream; @@ -47,7 +48,7 @@ import static io.prestosql.spi.type.BooleanType.BOOLEAN; import static java.util.Objects.requireNonNull; public class BooleanColumnReader - implements ColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(BooleanColumnReader.class).instanceSize(); @@ -224,4 +225,10 @@ public class BooleanColumnReader { return INSTANCE_SIZE + sizeOf(nullVector); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Byte value) + { + return filter.testBoolean(value != 0); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanSelectiveColumnReader.java index 503d931b7..dc5ab9574 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/BooleanSelectiveColumnReader.java @@ -46,7 +46,7 @@ import static io.prestosql.spi.type.BooleanType.BOOLEAN; import static java.util.Objects.requireNonNull; public class BooleanSelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(BooleanSelectiveColumnReader.class).instanceSize(); @@ -84,7 +84,7 @@ public class BooleanSelectiveColumnReader LocalMemoryContext systemMemoryContext) { requireNonNull(filter, "filter is null"); - checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + //checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); this.columnDesc = requireNonNull(columnDesc, "stream is null"); this.filter = filter.orElse(null); this.outputRequired = outputRequired; @@ -158,7 +158,7 @@ public class BooleanSelectiveColumnReader } @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { if (!rowGroupOpen) { @@ -171,7 +171,7 @@ public class BooleanSelectiveColumnReader ensureValuesCapacity(positionCount, nullsAllowed && presentStream != null); } - if (filter != null) { + if (this.filter != null) { ensureOutputPositionsCapacity(positionCount); } else { @@ -188,7 +188,7 @@ public class BooleanSelectiveColumnReader if (dataStream == null && presentStream != null) { streamPosition = readAllNulls(positions, positionCount); } - else if (filter == null) { + else if (this.filter == null) { streamPosition = readNoFilter(positions, positionCount); } else { @@ -211,7 +211,7 @@ public class BooleanSelectiveColumnReader } else { boolean value = dataStream.nextBit(); - if (filter == null || filter.testBoolean(value)) { + if (this.filter == null || this.filter.testBoolean(value)) { if (outputRequired) { values[outputPositionCount] = (byte) (value ? 1 : 0); if (nullsAllowed && presentStream != null) { @@ -440,7 +440,7 @@ public class BooleanSelectiveColumnReader } @Override - public Block mergeBlocks(List blocks, int positionCount) + public Block mergeBlocks(List> blocks, int positionCount) { byte[] valuesCopy = new byte[positionCount]; boolean[] nullsCopy = new boolean[positionCount]; diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/ByteColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/ByteColumnReader.java index 820c32080..81623abb8 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/ByteColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/ByteColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnMetadata; import io.prestosql.orc.stream.BooleanInputStream; @@ -225,4 +226,10 @@ public class ByteColumnReader { return INSTANCE_SIZE + sizeOf(nullVector); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Byte value) + { + return filter.testLong(value); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReader.java index 486f63c06..15748f078 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/ColumnReader.java @@ -13,15 +13,21 @@ */ package io.prestosql.orc.reader; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.spi.block.Block; import java.io.IOException; public interface ColumnReader - extends AbstractColumnReader + extends AbstractColumnReader //Fixme(nitin): merge { Block readBlock() throws IOException; void prepareNextRead(int batchSize); + + default boolean filterTest(TupleDomainFilter filter, T value) /* FixMe(Nitin): Remove Default, force all readers to enforce */ + { + throw new IllegalArgumentException("Unsupported type for pushdown"); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/DataCachingSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/DataCachingSelectiveColumnReader.java new file mode 100644 index 000000000..249ea47dd --- /dev/null +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/DataCachingSelectiveColumnReader.java @@ -0,0 +1,175 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.orc.reader; + +import com.google.common.cache.Cache; +import io.airlift.log.Logger; +import io.prestosql.orc.OrcColumn; +import io.prestosql.orc.OrcDataSourceId; +import io.prestosql.orc.OrcRowDataCacheKey; +import io.prestosql.orc.TupleDomainFilter; +import io.prestosql.orc.metadata.ColumnEncoding; +import io.prestosql.orc.metadata.ColumnMetadata; +import io.prestosql.orc.metadata.OrcColumnId; +import io.prestosql.orc.stream.InputStreamSources; +import io.prestosql.orc.stream.StreamSourceMeta; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.DictionaryBlock; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.BitSet; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public class DataCachingSelectiveColumnReader + implements SelectiveColumnReader +{ + private static final Logger log = Logger.get(CachingColumnReader.class); + + private final Cache cache; + private final ColumnReader delegate; + private final OrcColumn column; + private final OrcColumnId columnId; + + private OrcDataSourceId orcDataSourceId; + private long stripeOffset; + private long rowGroupOffset; + private int[] positions; + private int positionCount; + + private Block cachedBlock; + private Block resultBlock; + private boolean isDictionary; + + public DataCachingSelectiveColumnReader(ColumnReader delegate, OrcColumn column, + Cache cache) + { + this.delegate = delegate; + this.column = column; + this.columnId = column.getColumnId(); + this.cache = cache; + } + + @Override + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException + { + Block block = cachedBlock.getRegion(offset, positions[positionCount - 1] + 1); + this.isDictionary = false; + this.resultBlock = block; + + if (filter != null) { + this.positions = new int[positionCount]; + + /* traverse block and find matches */ + this.positionCount = block.filter(positions, positionCount, this.positions, (value) -> delegate.filterTest(filter, value)); + if (this.positionCount != positionCount) { + this.positions = Arrays.copyOf(this.positions, this.positionCount); /* since dictionary block uses arr.length */ + this.isDictionary = true; + } + } + else { + this.positions = positions.clone(); + this.positionCount = positionCount; + } + + return this.positionCount; + } + + @Override + public int readOr(int offset, int[] positions, int positionCount, List filter, BitSet accumulator) + { + Block block = cachedBlock.getRegion(offset, positions[positionCount - 1] + 1); + this.positionCount = positionCount; + this.positions = positions.clone(); + this.isDictionary = false; + + /* traverse block and find matches */ + for (int i = 0; i < positionCount; i++) { + if (accumulator.get(positions[i]) + || delegate.filterTest(filter.get(0), block.get(positions[i]))) { + accumulator.set(positions[i]); + } + } + + this.resultBlock = block; + return this.positionCount; + } + + @Override + public int[] getReadPositions() + { + return positions; + } + + @Override + public Block getBlock(int[] positions, int positionCount) + { + if (this.resultBlock.getPositionCount() != positionCount || this.isDictionary) { + return new DictionaryBlock(this.resultBlock, positions); + } + return this.resultBlock; + } + + @Override + public void startStripe(ZoneId fileTimeZone, ZoneId storageTimeZone, InputStreamSources dictionaryStreamSources, ColumnMetadata encoding) throws IOException + { + delegate.startStripe(fileTimeZone, storageTimeZone, dictionaryStreamSources, encoding); + } + + @Override + public void startRowGroup(InputStreamSources dataStreamSources) throws IOException + { + StreamSourceMeta streamSourceMeta = dataStreamSources.getStreamSourceMeta(); + orcDataSourceId = streamSourceMeta.getDataSourceId(); + stripeOffset = streamSourceMeta.getStripeOffset(); + rowGroupOffset = streamSourceMeta.getRowGroupOffset(); + cachedBlock = getCachedBlock(streamSourceMeta.getRowCount(), dataStreamSources); + //reset the stream - may not be required at all + delegate.startRowGroup(dataStreamSources); + } + + @Override + public void close() + { + delegate.close(); + } + + @Override + public long getRetainedSizeInBytes() + { + return delegate.getRetainedSizeInBytes(); + } + + private Block getCachedBlock(long rowCount, InputStreamSources dataStreamSources) throws IOException + { + OrcRowDataCacheKey cacheKey = new OrcRowDataCacheKey(); + cacheKey.setOrcDataSourceId(orcDataSourceId); + cacheKey.setStripeOffset(stripeOffset); + cacheKey.setRowGroupOffset(rowGroupOffset); + cacheKey.setColumnId(columnId); + try { + return cache.get(cacheKey, () -> { + delegate.startRowGroup(dataStreamSources); + delegate.prepareNextRead((int) rowCount); + log.debug("Caching row group data. DatasourceId = %s, columnId = %s, stripeOffset = %d, rowGroupOffset = %d, Column = %s", orcDataSourceId, columnId, stripeOffset, rowGroupOffset, column); + return delegate.readBlock(); + }); + } + catch (ExecutionException e) { + throw new IOException(e.getCause()); + } + } +} diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/DateColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/DateColumnReader.java index 81b5683ae..65d1d0218 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/DateColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/DateColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.RunLengthEncodedBlock; import io.prestosql.spi.type.DateType; @@ -92,4 +93,10 @@ public class DateColumnReader return block; } + + @Override + public boolean filterTest(TupleDomainFilter filter, Integer value) + { + return filter.testLong(value); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/DecimalColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/DecimalColumnReader.java index 16249aaf6..f519cc64d 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/DecimalColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/DecimalColumnReader.java @@ -18,6 +18,7 @@ import io.airlift.slice.Slices; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnMetadata; import io.prestosql.orc.stream.BooleanInputStream; @@ -360,4 +361,15 @@ public class DecimalColumnReader { return INSTANCE_SIZE + sizeOf(nullVector) + sizeOf(scaleVector); } + + @Override + public boolean filterTest(TupleDomainFilter filter, T value) + { + if (type.isShort()) { + return filter.testLong((Long) value); + } + + long[] data = (long[]) value; + return filter.testDecimal(data[0], data[1]); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleColumnReader.java index 9d856336c..0800177f4 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnMetadata; import io.prestosql.orc.stream.BooleanInputStream; @@ -47,8 +48,8 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour import static io.prestosql.spi.type.DoubleType.DOUBLE; import static java.util.Objects.requireNonNull; -public class DoubleColumnReader - implements ColumnReader +public class DoubleColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(DoubleColumnReader.class).instanceSize(); @@ -227,4 +228,10 @@ public class DoubleColumnReader { return INSTANCE_SIZE + sizeOf(nullVector); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Long value) + { + return filter.testDouble(Double.longBitsToDouble(value)); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleSelectiveColumnReader.java index e52a19a0d..dc4974977 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/DoubleSelectiveColumnReader.java @@ -48,7 +48,7 @@ import static java.lang.Double.doubleToLongBits; import static java.util.Objects.requireNonNull; public class DoubleSelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(DoubleSelectiveColumnReader.class).instanceSize(); @@ -368,7 +368,7 @@ public class DoubleSelectiveColumnReader } @Override - public Block getBlock(int[] positions, int positionCount) + public Block getBlock(int[] positions, int positionCount) { checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero"); checkState(outputRequired, "This stream reader doesn't produce output"); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/FloatColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/FloatColumnReader.java index c6e8473c3..c4e96c299 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/FloatColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/FloatColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnMetadata; import io.prestosql.orc.stream.BooleanInputStream; @@ -46,8 +47,8 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour import static io.prestosql.spi.type.RealType.REAL; import static java.util.Objects.requireNonNull; -public class FloatColumnReader - implements ColumnReader +public class FloatColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(FloatColumnReader.class).instanceSize(); @@ -226,4 +227,10 @@ public class FloatColumnReader { return INSTANCE_SIZE + sizeOf(nullVector); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Integer value) + { + return filter.testFloat(Float.intBitsToFloat(value)); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/FloatSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/FloatSelectiveColumnReader.java index aa57093da..ee195eee3 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/FloatSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/FloatSelectiveColumnReader.java @@ -46,7 +46,7 @@ import static java.lang.Float.floatToRawIntBits; import static java.util.Objects.requireNonNull; public class FloatSelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(FloatSelectiveColumnReader.class).instanceSize(); private static final Block NULL_BLOCK = REAL.createBlockBuilder(null, 1).appendNull().build(); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/IntegerColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/IntegerColumnReader.java index abe1ae593..b1b2087f5 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/IntegerColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/IntegerColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.IntArrayBlock; import io.prestosql.spi.block.RunLengthEncodedBlock; @@ -50,7 +51,7 @@ public class IntegerColumnReader } @Override - public Block readBlock() + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -133,4 +134,10 @@ public class IntegerColumnReader return new IntArrayBlock(nextBatchSize, Optional.of(isNull), result); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Integer value) + { + return filter.testLong(value); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/LongColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/LongColumnReader.java index 990aad686..d1249aa52 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/LongColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/LongColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.LongArrayBlock; import io.prestosql.spi.block.RunLengthEncodedBlock; @@ -49,7 +50,7 @@ public class LongColumnReader } @Override - public Block readBlock() + public Block readBlock() throws IOException { if (!rowGroupOpen) { @@ -70,7 +71,7 @@ public class LongColumnReader } } - Block block; + Block block; if (dataStream == null) { if (presentStream == null) { throw new OrcCorruptionException(column.getOrcDataSourceId(), "Value is null but present stream is missing"); @@ -132,4 +133,10 @@ public class LongColumnReader return new LongArrayBlock(nextBatchSize, Optional.of(isNull), result); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Long value) + { + return filter.testLong(value); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/LongDecimalSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/LongDecimalSelectiveColumnReader.java index 84a81b8f4..3c44a4b86 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/LongDecimalSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/LongDecimalSelectiveColumnReader.java @@ -33,7 +33,7 @@ import java.util.Optional; import static io.prestosql.spi.type.UnscaledDecimal128Arithmetic.rescale; public class LongDecimalSelectiveColumnReader - extends AbstractDecimalSelectiveColumnReader + extends AbstractDecimalSelectiveColumnReader { public LongDecimalSelectiveColumnReader( OrcType orcType, @@ -233,7 +233,7 @@ public class LongDecimalSelectiveColumnReader } @Override - protected Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values) + protected Block makeBlock(int positionCount, boolean includeNulls, boolean[] nulls, long[] values) { return new Int128ArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values); } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/LongSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/LongSelectiveColumnReader.java index 2e673e582..d89d67717 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/LongSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/LongSelectiveColumnReader.java @@ -44,7 +44,7 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour import static java.util.Objects.requireNonNull; public class LongSelectiveColumnReader - extends AbstractLongSelectiveColumnReader + extends AbstractLongSelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(LongSelectiveColumnReader.class).instanceSize(); @@ -87,7 +87,7 @@ public class LongSelectiveColumnReader // positions: Array of position from which data to be read; // positionCount: Number of position in the positions array. @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { if (!rowGroupOpen) { @@ -103,7 +103,7 @@ public class LongSelectiveColumnReader // If there is filter on this column then require space to store position. Number of position can not be more // positionCount - if (filter != null) { + if (this.filter != null) { ensureOutputPositionsCapacity(positionCount); } else { @@ -137,7 +137,7 @@ public class LongSelectiveColumnReader if (outputRequired) { nulls[outputPositionCount] = true; } - if (filter != null) { + if (this.filter != null) { outputPositions[outputPositionCount] = position; } outputPositionCount++; @@ -145,7 +145,7 @@ public class LongSelectiveColumnReader } else { long value = dataStream.next(); - if (filter == null || filter.testLong(value)) { + if (this.filter == null || this.filter.testLong(value)) { if (outputRequired) { values[outputPositionCount] = value; if (nullsAllowed && presentStream != null) { diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveCachingColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/ResultCachingSelectiveColumnReader.java similarity index 91% rename from presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveCachingColumnReader.java rename to presto-orc/src/main/java/io/prestosql/orc/reader/ResultCachingSelectiveColumnReader.java index c7fd7be67..694a6f140 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveCachingColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/ResultCachingSelectiveColumnReader.java @@ -36,8 +36,8 @@ import java.util.BitSet; import java.util.List; import java.util.function.BiConsumer; -public class SelectiveCachingColumnReader - implements SelectiveColumnReader +public class ResultCachingSelectiveColumnReader + implements SelectiveColumnReader { private static final Logger log = Logger.get(SelectiveColumnReader.class); private final Cache cache; @@ -52,12 +52,14 @@ public class SelectiveCachingColumnReader private int readSize; private int totalPositionCount; private Block cachedBlock; - private List accumulatorBlocks; + private int[] positions; + + private List> accumulatorBlocks; private OrcSelectiveRowDataCacheKey cacheKey; - public SelectiveCachingColumnReader(Cache cache, - SelectiveColumnReader delegate, OrcColumn column, - OrcPredicate predicate) + public ResultCachingSelectiveColumnReader(Cache cache, + SelectiveColumnReader delegate, OrcColumn column, + OrcPredicate predicate) { this.cache = cache; this.delegate = delegate; @@ -67,16 +69,17 @@ public class SelectiveCachingColumnReader } @Override - public int read(int offset, int[] positions, int positionCount) throws IOException + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { if (cachedBlock != null) { this.readSize = Integer.min(positionCount, (cachedBlock.getPositionCount() - this.offset)); this.offset += this.readSize; + this.positions = positions; return this.readSize; } - return delegate.read(offset, positions, positionCount); + return delegate.read(offset, positions, positionCount, null); } @Override @@ -86,6 +89,7 @@ public class SelectiveCachingColumnReader if (cachedBlock != null) { this.readSize = Integer.min(positionCount, (cachedBlock.getPositionCount() - this.offset)); this.offset += this.readSize; + this.positions = positions; if (this.readSize > 0) { accumulator.set(offset, offset + this.readSize); @@ -99,6 +103,10 @@ public class SelectiveCachingColumnReader @Override public int[] getReadPositions() { + if (cachedBlock != null) { + return this.positions; + } + return delegate.getReadPositions(); } @@ -192,7 +200,7 @@ public class SelectiveCachingColumnReader return delegate.getRetainedSizeInBytes(); } - private Block mergeAccumulatedBlocks(List accumulatedBlocks) + private Block mergeAccumulatedBlocks(List> accumulatedBlocks) { /* Fixme(Nitin): merge all accumulated blocks in single array */ if (accumulatedBlocks.size() > 0) { @@ -201,7 +209,7 @@ public class SelectiveCachingColumnReader totalPositionCount); } - return new Block() { + return new Block() { @Override public void writePositionTo(int position, BlockBuilder blockBuilder) { diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReader.java index 840fedadd..7799e84be 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReader.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.BitSet; import java.util.List; -public interface SelectiveColumnReader +public interface SelectiveColumnReader extends AbstractColumnReader { /** @@ -30,9 +30,10 @@ public interface SelectiveColumnReader ** @param positions Monotonically increasing positions to read ** @param positionCount Number of valid positions in the positions array; may be less than the ** size of the array - ** @return the number of positions that passed the filter + ** @param filter + * @return the number of positions that passed the filter **/ - int read(int offset, int[] positions, int positionCount) + int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException; /** @@ -49,15 +50,15 @@ public interface SelectiveColumnReader ** from getReadPositions() ** @param positionCount Number of valid positions in the positions array; may be less than the **/ - Block getBlock(int[] positions, int positionCount); + Block getBlock(int[] positions, int positionCount); default int readOr(int offset, int[] positions, int positionCount, List filter, BitSet accumulator) throws IOException { - return read(offset, positions, positionCount); + return read(offset, positions, positionCount, null); } - default Block mergeBlocks(List blocks, int positionCount) + default Block mergeBlocks(List> blocks, int positionCount) { return blocks.get(0); } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReaders.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReaders.java index 72d28fef7..839810ecb 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReaders.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SelectiveColumnReaders.java @@ -57,12 +57,14 @@ public final class SelectiveColumnReaders return new TimestampSelectiveColumnReader(column, filter, hiveStorageTimeZone, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName())); case DECIMAL: if (orcType.getPrecision().get() > MAX_SHORT_PRECISION) { - throw new IllegalArgumentException("Unsupported type: LONG DECIMAL"); + return new LongDecimalSelectiveColumnReader(orcType, column, filter, outputType, systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName())); } return new ShortDecimalSelectiveColumnReader(orcType, column, filter, outputType, systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName())); - case BYTE: - case FLOAT: case DOUBLE: + return new DoubleSelectiveColumnReader(column, filter, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveColumnReaders.class.getSimpleName())); + case FLOAT: + return new FloatSelectiveColumnReader(column, filter, outputType.isPresent(), systemMemoryContext.newLocalMemoryContext(SelectiveColumnReader.class.getSimpleName())); + case BYTE: case LIST: case STRUCT: case UNION: @@ -71,11 +73,18 @@ public final class SelectiveColumnReaders } } - public static SelectiveColumnReader wrapWithCachingStreamReader(SelectiveColumnReader original, - OrcColumn column, - OrcPredicate predicate, - Cache cache) + public static SelectiveColumnReader wrapWithResultCachingStreamReader(SelectiveColumnReader original, + OrcColumn column, + OrcPredicate predicate, + Cache cache) { - return new SelectiveCachingColumnReader(cache, original, column, predicate); + return new ResultCachingSelectiveColumnReader(cache, original, column, predicate); + } + + public static SelectiveColumnReader wrapWithDataCachingStreamReader(ColumnReader original, + OrcColumn column, + Cache cache) + { + return new DataCachingSelectiveColumnReader(original, column, cache); } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/ShortColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/ShortColumnReader.java index 8ba6ad060..a8b79a72a 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/ShortColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/ShortColumnReader.java @@ -16,6 +16,7 @@ package io.prestosql.orc.reader; import io.prestosql.memory.context.LocalMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.RunLengthEncodedBlock; import io.prestosql.spi.block.ShortArrayBlock; @@ -133,4 +134,10 @@ public class ShortColumnReader return new ShortArrayBlock(nextBatchSize, Optional.of(isNull), result); } + + @Override + public boolean filterTest(TupleDomainFilter filter, Short value) + { + return filter.testLong(value); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/ShortDecimalSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/ShortDecimalSelectiveColumnReader.java index 4f0c91a05..3af81aa70 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/ShortDecimalSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/ShortDecimalSelectiveColumnReader.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Optional; public class ShortDecimalSelectiveColumnReader - extends AbstractDecimalSelectiveColumnReader + extends AbstractDecimalSelectiveColumnReader { public ShortDecimalSelectiveColumnReader( OrcType orcType, @@ -216,7 +216,7 @@ public class ShortDecimalSelectiveColumnReader } @Override - public Block mergeBlocks(List blocks, int positionCount) + public Block mergeBlocks(List> blocks, int positionCount) { LongArrayBlockBuilder blockBuilder = new LongArrayBlockBuilder(null, positionCount); blocks.stream().forEach(block -> { diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java index 4e99cceda..cb84d99b8 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceColumnReader.java @@ -18,6 +18,7 @@ import io.airlift.slice.Slice; import io.prestosql.memory.context.AggregatedMemoryContext; import io.prestosql.orc.OrcColumn; import io.prestosql.orc.OrcCorruptionException; +import io.prestosql.orc.TupleDomainFilter; import io.prestosql.orc.metadata.ColumnEncoding; import io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind; import io.prestosql.orc.metadata.ColumnMetadata; @@ -46,8 +47,8 @@ import static io.prestosql.spi.type.Varchars.byteCount; import static io.prestosql.spi.type.Varchars.isVarcharType; import static java.util.Objects.requireNonNull; -public class SliceColumnReader - implements ColumnReader +public class SliceColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceColumnReader.class).instanceSize(); @@ -160,4 +161,10 @@ public class SliceColumnReader { return INSTANCE_SIZE + directReader.getRetainedSizeInBytes() + dictionaryReader.getRetainedSizeInBytes(); } + + @Override + public boolean filterTest(TupleDomainFilter filter, byte[] value) + { + return filter.testBytes(value, 0, value.length); + } } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionaryColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionaryColumnReader.java index 65590395c..2956ca7ec 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionaryColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionaryColumnReader.java @@ -52,8 +52,8 @@ import static java.lang.Math.toIntExact; import static java.util.Arrays.fill; import static java.util.Objects.requireNonNull; -public class SliceDictionaryColumnReader - implements ColumnReader +public class SliceDictionaryColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDictionaryColumnReader.class).instanceSize(); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionarySelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionarySelectiveColumnReader.java index 85a671131..a7c8b8e80 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionarySelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDictionarySelectiveColumnReader.java @@ -31,7 +31,6 @@ import io.prestosql.spi.block.Block; import io.prestosql.spi.block.DictionaryBlock; import io.prestosql.spi.block.RunLengthEncodedBlock; import io.prestosql.spi.block.VariableWidthBlock; -import io.prestosql.spi.type.Chars; import io.prestosql.spi.type.Type; import org.openjdk.jol.info.ClassLayout; @@ -59,7 +58,7 @@ import static java.lang.Math.toIntExact; import static java.util.Objects.requireNonNull; public class SliceDictionarySelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDictionarySelectiveColumnReader.class).instanceSize(); @@ -120,15 +119,15 @@ public class SliceDictionarySelectiveColumnReader this.isCharType = orcType.getOrcTypeKind() == CHAR; this.outputRequired = outputType.isPresent(); this.orcType = orcType; - checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + //checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); } @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { return readOr(offset, positions, positionCount, - (filter == null) ? null : ImmutableList.of(filter), + (this.filter == null) ? null : ImmutableList.of(this.filter), null); } @@ -229,18 +228,10 @@ public class SliceDictionarySelectiveColumnReader } else { int index = toIntExact(dataStream.next()); - int length = stripeDictionaryLength[index]; - - if (true) { /* Fixme(Nitin): Why this condition here? - (accumulator != null && accumulator.get(position)) || filters == null - */ - int currentPosLength = dictionaryBlock.getSliceLength(index); + int currentPosLength = dictionaryBlock.getSliceLength(index); + if (filters.get(0).testLength(currentPosLength)) { Slice data = dictionaryBlock.getSlice(index, 0, currentPosLength); - if (isCharType) { - data = Chars.padSpaces(data, maxCodePointCount); - } - Slice finalData = data; - if (filters == null || filters.get(0).testBytes(finalData.getBytes(), 0, length)) { + if (filters == null || filters.get(0).testBytes(data.getBytes(), 0, currentPosLength)) { if (outputRequired) { values[outputPositionCount] = index; } @@ -277,19 +268,11 @@ public class SliceDictionarySelectiveColumnReader } else { int index = toIntExact(dataStream.next()); - int length = stripeDictionaryLength[index]; - - if (true) { /* Fixme(Nitin): Why this condition here? - (accumulator != null && accumulator.get(position)) || filters == null - */ - int currentPosLength = dictionaryBlock.getSliceLength(index); + int currentPosLength = dictionaryBlock.getSliceLength(index); + if (filters.get(0).testLength(currentPosLength)) { Slice data = dictionaryBlock.getSlice(index, 0, currentPosLength); - if (isCharType) { - data = Chars.padSpaces(data, maxCodePointCount); - } - Slice finalData = data; if ((accumulator != null && accumulator.get(position)) - || filters == null || filters.get(0).testBytes(finalData.getBytes(), 0, length)) { + || filters == null || filters.get(0).testBytes(data.getBytes(), 0, currentPosLength)) { if (accumulator != null) { accumulator.set(position); } diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectColumnReader.java index 69e8f0151..1504bfe2e 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectColumnReader.java @@ -54,8 +54,8 @@ import static java.lang.Math.toIntExact; import static java.lang.String.format; import static java.util.Objects.requireNonNull; -public class SliceDirectColumnReader - implements ColumnReader +public class SliceDirectColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDirectColumnReader.class).instanceSize(); private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes()); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectSelectiveColumnReader.java index 5ffde50a5..5d2743806 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceDirectSelectiveColumnReader.java @@ -57,7 +57,7 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class SliceDirectSelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceDirectSelectiveColumnReader.class).instanceSize(); private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes()); @@ -109,15 +109,15 @@ public class SliceDirectSelectiveColumnReader this.orcType = orcType; this.isCharType = orcType.getOrcTypeKind() == OrcType.OrcTypeKind.CHAR; this.maxCodePointCount = orcType.getLength().orElse(-1); - checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); + //checkArgument(filter.isPresent() || outputRequired, "filter must be present if outputRequired is false"); } @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { return readOr(offset, positions, positionCount, - (filter == null) ? null : ImmutableList.of(filter), + (this.filter == null) ? null : ImmutableList.of(this.filter), null); } @@ -236,41 +236,40 @@ public class SliceDirectSelectiveColumnReader else { int length = lengthVector[lengthIndex]; int dataOffset = outputRequired ? offset : 0; - if (true) { /* Fixme(Nitin): Why this check is needed? */ + if (true /* filters.get(0).testLength(length) */) { if (dataStream != null) { dataStream.skip(dataToSkip); dataToSkip = 0; dataStream.next(data, dataOffset, dataOffset + length); - if ((accumulator != null && accumulator.get(position)) || filters == null || filters.get(0).testBytes(data, dataOffset, length)) { if (accumulator != null) { accumulator.set(position); } } - - if (outputRequired) { - int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType); - offsets[outputPositionCount + 1] = offset + truncatedLength; - if (nullsAllowed && isNullVector != null) { - nulls[outputPositionCount] = false; - } - } - outputPositions[outputPositionCount] = position; - outputPositionCount++; } else { - if (outputRequired) { - offsets[outputPositionCount + 1] = offset; - if (nullsAllowed && isNullVector != null) { - nulls[outputPositionCount] = false; + if ((accumulator != null && accumulator.get(position)) + || filters == null || filters.get(0).testBytes("".getBytes(), 0, 0)) { + if (accumulator != null) { + accumulator.set(position); } } - - /* Fixme(Rajeev): This may also need to evaluate filter condition? */ - outputPositions[outputPositionCount] = position; - outputPositionCount++; } + + if (outputRequired) { + int truncatedLength = 0; + if (dataStream != null) { + truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType); + } + + offsets[outputPositionCount + 1] = offset + truncatedLength; + if (nullsAllowed && isNullVector != null) { + nulls[outputPositionCount] = false; + } + } + outputPositions[outputPositionCount] = position; + outputPositionCount++; } else { dataToSkip += length; @@ -316,15 +315,14 @@ public class SliceDirectSelectiveColumnReader else { int length = lengthVector[lengthIndex]; int dataOffset = outputRequired ? offset : 0; - if (true) { /* Fixme(Nitin): Why this check is needed? */ + if (isCharType || filters.get(0).testLength(length)) { if (dataStream != null) { dataStream.skip(dataToSkip); dataToSkip = 0; dataStream.next(data, dataOffset, dataOffset + length); - - if (filters == null || filters.get(0).testBytes(data, dataOffset, length)) { + int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType); + if (filters == null || filters.get(0).testBytes(data, dataOffset, truncatedLength)) { if (outputRequired) { - int truncatedLength = computeTruncatedLength(dataAsSlice, dataOffset, length, maxCodePointCount, isCharType); offsets[outputPositionCount + 1] = offset + truncatedLength; if (nullsAllowed && isNullVector != null) { nulls[outputPositionCount] = false; @@ -335,16 +333,16 @@ public class SliceDirectSelectiveColumnReader } } else { - if (outputRequired) { - offsets[outputPositionCount + 1] = offset; - if (nullsAllowed && isNullVector != null) { - nulls[outputPositionCount] = false; + if (filters == null || filters.get(0).testBytes("".getBytes(), 0, 0)) { + if (outputRequired) { + offsets[outputPositionCount + 1] = offset; + if (nullsAllowed && isNullVector != null) { + nulls[outputPositionCount] = false; + } } + outputPositions[outputPositionCount] = position; + outputPositionCount++; } - - /* Fixme(Rajeev): This may also need to evaluate filter condition? */ - outputPositions[outputPositionCount] = position; - outputPositionCount++; } } else { diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceSelectiveColumnReader.java index 136ad2d5a..1cce01029 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/SliceSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/SliceSelectiveColumnReader.java @@ -43,7 +43,7 @@ import static io.prestosql.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT import static java.util.Objects.requireNonNull; public class SliceSelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(SliceSelectiveColumnReader.class).instanceSize(); @@ -111,10 +111,10 @@ public class SliceSelectiveColumnReader } @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { - return currentReader.read(offset, positions, positionCount); + return currentReader.read(offset, positions, positionCount, null); } @Override @@ -135,18 +135,6 @@ public class SliceSelectiveColumnReader return currentReader.getBlock(positions, positionCount); } -// @Override -// public BlockLease getBlockView(int[] positions, int positionCount) -// { -// return currentReader.getBlockView(positions, positionCount); -// } -// -// @Override -// public void throwAnyError(int[] positions, int positionCount) -// { -// currentReader.throwAnyError(positions, positionCount); -// } - public static int computeTruncatedLength(Slice slice, int offset, int length, int maxCodePointCount, boolean isCharType) { if (isCharType) { diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampColumnReader.java index ac264d8d6..87f086029 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampColumnReader.java @@ -49,8 +49,8 @@ import static io.prestosql.orc.stream.MissingInputStreamSource.missingStreamSour import static io.prestosql.spi.type.TimestampType.TIMESTAMP; import static java.util.Objects.requireNonNull; -public class TimestampColumnReader - implements ColumnReader +public class TimestampColumnReader + implements ColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(TimestampColumnReader.class).instanceSize(); diff --git a/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampSelectiveColumnReader.java b/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampSelectiveColumnReader.java index 9fcafcdfe..b1d9923ec 100644 --- a/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampSelectiveColumnReader.java +++ b/presto-orc/src/main/java/io/prestosql/orc/reader/TimestampSelectiveColumnReader.java @@ -51,7 +51,7 @@ import static io.prestosql.spi.type.TimestampType.TIMESTAMP; import static java.util.Objects.requireNonNull; public class TimestampSelectiveColumnReader - implements SelectiveColumnReader + implements SelectiveColumnReader { private static final int INSTANCE_SIZE = ClassLayout.parseClass(TimestampSelectiveColumnReader.class).instanceSize(); private static final Block NULL_BLOCK = TIMESTAMP.createBlockBuilder(null, 1).appendNull().build(); @@ -141,11 +141,11 @@ public class TimestampSelectiveColumnReader } @Override - public int read(int offset, int[] positions, int positionCount) + public int read(int offset, int[] positions, int positionCount, TupleDomainFilter filter) throws IOException { return readOr(offset, positions, positionCount, - (filter == null) ? null : ImmutableList.of(filter), + (this.filter == null) ? null : ImmutableList.of(this.filter), null); } @@ -426,7 +426,7 @@ public class TimestampSelectiveColumnReader } @Override - public Block mergeBlocks(List blocks, int positionCount) + public Block mergeBlocks(List> blocks, int positionCount) { LongArrayBlockBuilder blockBuilder = new LongArrayBlockBuilder(null, positionCount); blocks.stream().forEach(block -> { diff --git a/presto-orc/src/test/java/io/prestosql/orc/BenchmarkSelectiveColumnReaders.java b/presto-orc/src/test/java/io/prestosql/orc/BenchmarkSelectiveColumnReaders.java index 2c7da0145..05b4c8914 100644 --- a/presto-orc/src/test/java/io/prestosql/orc/BenchmarkSelectiveColumnReaders.java +++ b/presto-orc/src/test/java/io/prestosql/orc/BenchmarkSelectiveColumnReaders.java @@ -181,7 +181,10 @@ public class BenchmarkSelectiveColumnReaders new OrcCacheProperties(), Optional.empty(), new HashMap<>(), - new ArrayList<>()); + new ArrayList<>(), + false, + ImmutableMap.of(), + ImmutableMap.of()); } } diff --git a/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java b/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java index 22b03dcc9..008acc07d 100644 --- a/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java +++ b/presto-orc/src/test/java/io/prestosql/orc/OrcTester.java @@ -559,7 +559,10 @@ public class OrcTester new OrcCacheProperties(), Optional.empty(), new HashMap<>(), - null); + null, + false, + ImmutableMap.of(), + ImmutableMap.of()); } private static List makeColumnNames(int columns) @@ -611,6 +614,9 @@ public class OrcTester return false; } } + else if (type == TIMESTAMP) { + return filter.testLong(((SqlTimestamp) value).getMillisUtc()); + } else if (type == VARCHAR) { return filter.testBytes(((String) value).getBytes(), 0, ((String) value).length()); } @@ -629,6 +635,11 @@ public class OrcTester return filter.testLong(bigDecimal.unscaledValue().longValue()); } } + else if (type == DOUBLE) { + if (!filter.testDouble((double) value)) { + return false; + } + } else { fail("Unsupported type: " + type); } @@ -1377,7 +1388,7 @@ public class OrcTester return METADATA.getParameterizedType(StandardTypes.ARRAY, ImmutableList.of(TypeSignatureParameter.of(elementType.getTypeSignature()))); } - private static Type mapType(Type keyType, Type valueType) + public static Type mapType(Type keyType, Type valueType) { return METADATA.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature()))); } diff --git a/presto-orc/src/test/java/io/prestosql/orc/TestSelectiveOrcReader.java b/presto-orc/src/test/java/io/prestosql/orc/TestSelectiveOrcReader.java index 9b8282b78..0c4d44697 100644 --- a/presto-orc/src/test/java/io/prestosql/orc/TestSelectiveOrcReader.java +++ b/presto-orc/src/test/java/io/prestosql/orc/TestSelectiveOrcReader.java @@ -26,6 +26,7 @@ import io.prestosql.spi.type.CharType; import io.prestosql.spi.type.DecimalType; import io.prestosql.spi.type.SqlDate; import io.prestosql.spi.type.SqlDecimal; +import io.prestosql.spi.type.SqlTimestamp; import org.joda.time.DateTimeZone; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -37,22 +38,30 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.function.Function; import java.util.stream.IntStream; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.cycle; import static com.google.common.collect.Iterables.limit; import static com.google.common.collect.Lists.newArrayList; import static io.prestosql.orc.OrcTester.HIVE_STORAGE_TIME_ZONE; +import static io.prestosql.orc.OrcTester.mapType; import static io.prestosql.orc.OrcTester.quickSelectiveOrcTester; +import static io.prestosql.orc.TupleDomainFilter.IS_NOT_NULL; import static io.prestosql.orc.TupleDomainFilter.IS_NULL; import static io.prestosql.spi.type.BigintType.BIGINT; import static io.prestosql.spi.type.BooleanType.BOOLEAN; import static io.prestosql.spi.type.CharType.createCharType; import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DoubleType.DOUBLE; import static io.prestosql.spi.type.IntegerType.INTEGER; import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static io.prestosql.testing.DateTimeTestingUtils.sqlTimestampOf; +import static io.prestosql.testing.TestingConnectorSession.SESSION; import static java.util.Collections.nCopies; import static java.util.stream.Collectors.toList; import static org.testng.Assert.assertEquals; @@ -63,6 +72,7 @@ public class TestSelectiveOrcReader private static final DecimalType DECIMAL_TYPE_PRECISION_2 = DecimalType.createDecimalType(2, 1); private static final DecimalType DECIMAL_TYPE_PRECISION_4 = DecimalType.createDecimalType(4, 2); + private static final DecimalType DECIMAL_TYPE_PRECISION_20 = DecimalType.createDecimalType(20, 2); private static final CharType CHAR_10 = createCharType(10); @BeforeClass @@ -91,7 +101,12 @@ public class TestSelectiveOrcReader public void testLongSequenceWithHoles() throws Exception { - testRoundTripNumeric(skipEvery(5, intsBetween(0, 31_234)), BigintRange.of(10, 100, false)); + BigintRange filters1 = BigintRange.of(10, 100, false); + BigintRange filters2 = BigintRange.of(200, 300, false); + testRoundTripNumeric(intsBetween(0, 100), TupleDomainFilter.BigintMultiRange.of(ImmutableList.of(filters1, filters2), true)); + long[] values = new long[]{1, 2, 3, 4}; + TupleDomainFilter.BigintValues filter3 = TupleDomainFilter.BigintValues.of(values, true); + testRoundTripNumeric(intsBetween(0, 100), filter3); } @Test @@ -123,6 +138,10 @@ public class TestSelectiveOrcReader .map(SqlDate::new) .collect(toList()); + List timestamps = longValues.stream() + .map(timestamp -> sqlTimestampOf(timestamp & Integer.MAX_VALUE, SESSION)) + .collect(toList()); + tester.testRoundTrip(BIGINT, longValues, ImmutableList.of(ImmutableMap.of(0, filter))); tester.testRoundTrip(INTEGER, intValues, ImmutableList.of(ImmutableMap.of(0, filter))); @@ -131,6 +150,8 @@ public class TestSelectiveOrcReader tester.testRoundTrip(DATE, dateValues, ImmutableList.of(ImmutableMap.of(0, filter))); + tester.testRoundTrip(TIMESTAMP, timestamps, ImmutableList.of(ImmutableMap.of(0, filter))); + List reversedIntValues = new ArrayList<>(intValues); Collections.reverse(reversedIntValues); @@ -248,6 +269,24 @@ public class TestSelectiveOrcReader tester.testRoundTrip(DECIMAL_TYPE_PRECISION_2, decimalSequence("-30", "1", 60, 2, 1), filters); } + @Test(expectedExceptions = IllegalArgumentException.class) + public void testUnknowType() + throws Exception + { + tester.testRoundTrip(mapType(INTEGER, INTEGER), createList(1, i -> createMap(1))); + } + + private static List createList(int size, Function createElement) + { + return IntStream.range(0, size).mapToObj(createElement::apply).collect(toList()); + } + + private static Map createMap(int seed) + { + int mapSize = Math.abs(seed) % 7 + 1; + return IntStream.range(0, mapSize).boxed().collect(toImmutableMap(Function.identity(), i -> i + seed)); + } + private static List decimalSequence(String start, String step, int items, int precision, int scale) { BigInteger decimalStep = new BigInteger(step); @@ -259,4 +298,43 @@ public class TestSelectiveOrcReader } return values; } + + @Test + public void testDoubleSequence() + throws Exception + { + List> filters = ImmutableList.of( + ImmutableMap.of(0, TupleDomainFilter.DoubleRange.of(0, false, false, 1_000, false, false, false)), + ImmutableMap.of(0, IS_NULL), + ImmutableMap.of(0, IS_NOT_NULL)); + + tester.testRoundTrip(DOUBLE, doubleSequence(0, 0.1, 20), filters); + } + + @Test + public void testDoubleNaNInfinity() + throws Exception + { + List> filters = ImmutableList.of( + ImmutableMap.of(0, TupleDomainFilter.DoubleRange.of(0, false, false, 1_000, false, false, false)), + ImmutableMap.of(0, IS_NULL), + ImmutableMap.of(0, IS_NOT_NULL)); + + tester.testRoundTrip(DOUBLE, ImmutableList.of(1000.0, -1.0, Double.POSITIVE_INFINITY), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(-1000.0, Double.NEGATIVE_INFINITY, 1.0), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(0.0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), filters); + + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, -1.0, 1.0), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, -1.0, Double.POSITIVE_INFINITY), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, Double.NEGATIVE_INFINITY, 1.0), filters); + tester.testRoundTrip(DOUBLE, ImmutableList.of(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), filters); + } + + private static List doubleSequence(double start, double step, int items) + { + return IntStream.range(0, items) + .mapToDouble(i -> start + i * step) + .boxed() + .collect(ImmutableList.toImmutableList()); + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/Block.java b/presto-spi/src/main/java/io/prestosql/spi/block/Block.java index 8cff6fd16..8de8a3a26 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/Block.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/Block.java @@ -18,6 +18,7 @@ import io.prestosql.spi.util.BloomFilter; import sun.reflect.generics.reflectiveObjects.NotImplementedException; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; import static io.prestosql.spi.block.DictionaryId.randomDictionaryId; @@ -87,7 +88,7 @@ public interface Block /** * Gets an object in the value at {@code position}. */ - default T getObject(int position, Class clazz) + default R getObject(int position, Class clazz) { throw new UnsupportedOperationException(getClass().getName()); } @@ -333,4 +334,10 @@ public interface Block //by default we will not filter anything return validPositions; } + + default int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + System.arraycopy(positions, positionCount, matchedPositions, positionCount, positionCount); + return positionCount; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/BlockListBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/BlockListBlock.java index d134dfdaa..72359e160 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/BlockListBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/BlockListBlock.java @@ -21,8 +21,8 @@ import java.util.function.BiConsumer; import static io.airlift.slice.SizeOf.sizeOf; import static java.util.Objects.requireNonNull; -public class BlockListBlock - implements Block +public class BlockListBlock + implements Block { private static final int INSTANCE_SIZE = ClassLayout.parseClass(ArrayBlock.class).instanceSize(); private final Block[] blocks; @@ -197,20 +197,22 @@ public class BlockListBlock int blockIdx = lookupBlockForPosition(positionOffset); Block[] newBlocks = new Block[blockCount]; int newBlockCount = 0; - int newPositionCount = length; + int newPositionCount = 0; Block block; - for (int i = blockIdx; i < blockCount && newPositionCount > 0; i++, newBlockCount++) { - block = blocks[blockIdx]; + for (int i = blockIdx; i < blockCount && length > 0; i++) { + block = blocks[i]; /* if region is within the block return the block else wrap the candidate blocks */ if (block.getPositionCount() > (positionOffset - positionMap[blockIdx]) + length) { - newBlocks[i] = block.getRegion(positionOffset - positionMap[blockIdx], length); - newPositionCount -= length; + newBlocks[newBlockCount++] = block.getRegion(positionOffset - positionMap[blockIdx], length); + newPositionCount += length; + length -= length; } else { - newBlocks[i] = block; - newPositionCount -= block.getPositionCount(); + newBlocks[newBlockCount++] = block; + newPositionCount += block.getPositionCount(); + length -= block.getPositionCount(); } } @@ -290,10 +292,10 @@ public class BlockListBlock } @Override - public T getObject(int position, Class clazz) + public R getObject(int position, Class clazz) { int blockIdx = lookupBlockForPosition(position); - return blocks[blockIdx].getObject(position - positionMap[blockIdx], clazz); + return (R) blocks[blockIdx].getObject(position - positionMap[blockIdx], clazz); } @Override @@ -311,4 +313,11 @@ public class BlockListBlock return blocks[blockIdx].bytesCompare(position - positionMap[blockIdx], offset, length, otherSlice, otherOffset, otherLength); } + + @Override + public T get(int position) + { + int blockIdx = lookupBlockForPosition(position); + return (T) blocks[blockIdx].get(position - positionMap[blockIdx]); + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/ByteArrayBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/ByteArrayBlock.java index 9bb90fc0d..af2194bf8 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/ByteArrayBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/ByteArrayBlock.java @@ -20,6 +20,7 @@ import javax.annotation.Nullable; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.airlift.slice.SizeOf.sizeOf; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; @@ -27,8 +28,8 @@ import static io.prestosql.spi.block.BlockUtil.checkValidRegion; import static io.prestosql.spi.block.BlockUtil.compactArray; import static io.prestosql.spi.block.BlockUtil.countUsedPositions; -public class ByteArrayBlock - implements Block +public class ByteArrayBlock + implements Block { private static final int INSTANCE_SIZE = ClassLayout.parseClass(ByteArrayBlock.class).instanceSize(); @@ -233,4 +234,32 @@ public class ByteArrayBlock } return validPositions; } + + @Override + public int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + checkReadablePosition(positions[positionCount - 1]); + + int matchCount = 0; + for (int i = 0; i < positionCount; i++) { + if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) { + continue; + } + if (test.apply(values[positions[i] + arrayOffset])) { + matchedPositions[matchCount++] = positions[i]; + } + } + + return matchCount; + } + + @Override + public Byte get(int position) + { + if (valueIsNull != null && valueIsNull[position + arrayOffset]) { + return null; + } + + return values[position + arrayOffset]; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/Int128ArrayBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/Int128ArrayBlock.java index b1a1b7c7a..9276b6912 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/Int128ArrayBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/Int128ArrayBlock.java @@ -22,6 +22,7 @@ import javax.annotation.Nullable; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.airlift.slice.SizeOf.sizeOf; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; @@ -30,7 +31,7 @@ import static io.prestosql.spi.block.BlockUtil.compactArray; import static io.prestosql.spi.block.BlockUtil.countUsedPositions; public class Int128ArrayBlock - implements Block + implements Block { private static final int INSTANCE_SIZE = ClassLayout.parseClass(Int128ArrayBlock.class).instanceSize(); public static final int INT128_BYTES = Long.BYTES + Long.BYTES; @@ -244,4 +245,36 @@ public class Int128ArrayBlock } return validPositions; } + + @Override + public int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + int matchCount = 0; + long[] val = new long[2]; + for (int i = 0; i < positionCount; i++) { + if (valueIsNull != null && valueIsNull[positions[i] + positionOffset]) { + continue; + } + val[0] = values[(positions[i] + positionOffset) * 2]; + val[1] = values[((positions[i] + positionOffset) * 2) + 1]; + if (test.apply(val)) { + matchedPositions[matchCount++] = positions[i]; + } + } + + return matchCount; + } + + @Override + public long[] get(int position) + { + long[] val = new long[2]; + if (valueIsNull != null && valueIsNull[position + positionOffset]) { + return null; + } + val[0] = values[(position + positionOffset) * 2]; + val[1] = values[((position + positionOffset) * 2) + 1]; + + return val; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java index ed7ce4ec1..c49b82abe 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/IntArrayBlock.java @@ -20,6 +20,7 @@ import javax.annotation.Nullable; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.airlift.slice.SizeOf.sizeOf; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; @@ -239,4 +240,30 @@ public class IntArrayBlock } return validPositions; } + + @Override + public int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + int matchCount = 0; + for (int i = 0; i < positionCount; i++) { + if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) { + continue; + } + if (test.apply(values[positions[i] + arrayOffset])) { + matchedPositions[matchCount++] = positions[i]; + } + } + + return matchCount; + } + + @Override + public Integer get(int position) + { + if (valueIsNull != null && valueIsNull[position + arrayOffset]) { + return null; + } + + return values[position + arrayOffset]; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/LongArrayBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/LongArrayBlock.java index 19f9ab7b2..2b4b030c2 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/LongArrayBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/LongArrayBlock.java @@ -20,6 +20,7 @@ import javax.annotation.Nullable; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.airlift.slice.SizeOf.sizeOf; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; @@ -133,6 +134,9 @@ public class LongArrayBlock public Long get(int position) { + if (valueIsNull != null && valueIsNull[position + arrayOffset]) { + return null; + } return values[position + arrayOffset]; } @@ -283,9 +287,25 @@ public class LongArrayBlock @Override public boolean[] filter(BloomFilter filter, boolean[] validPositions) { - for (int i = 0; i < values.length; i++) { + for (int i = arrayOffset; i < positionCount; i++) { validPositions[i] = validPositions[i] && filter.test(values[i]); } return validPositions; } + + @Override + public int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + int matchCount = 0; + for (int i = 0; i < positionCount; i++) { + if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) { + continue; + } + if (test.apply(values[positions[i] + arrayOffset])) { + matchedPositions[matchCount++] = positions[i]; + } + } + + return matchCount; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/ShortArrayBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/ShortArrayBlock.java index b2224c3eb..e63c8b441 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/ShortArrayBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/ShortArrayBlock.java @@ -20,6 +20,7 @@ import javax.annotation.Nullable; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.airlift.slice.SizeOf.sizeOf; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; @@ -233,4 +234,30 @@ public class ShortArrayBlock } return validPositions; } + + @Override + public int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + int matchCount = 0; + for (int i = 0; i < positionCount; i++) { + if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) { + continue; + } + if (test.apply(values[positions[i] + arrayOffset])) { + matchedPositions[matchCount++] = positions[i]; + } + } + + return matchCount; + } + + @Override + public Short get(int position) + { + if (valueIsNull != null && valueIsNull[position + arrayOffset]) { + return null; + } + + return values[position + arrayOffset]; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlock.java b/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlock.java index 54e9ff5ba..c0f98e005 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlock.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlock.java @@ -23,6 +23,7 @@ import javax.annotation.Nullable; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.Function; import static io.airlift.slice.SizeOf.sizeOf; import static io.prestosql.spi.block.BlockUtil.checkArrayRange; @@ -31,8 +32,8 @@ import static io.prestosql.spi.block.BlockUtil.compactArray; import static io.prestosql.spi.block.BlockUtil.compactOffsets; import static io.prestosql.spi.block.BlockUtil.compactSlice; -public class VariableWidthBlock - extends AbstractVariableWidthBlock +public class VariableWidthBlock + extends AbstractVariableWidthBlock { private static final int INSTANCE_SIZE = ClassLayout.parseClass(VariableWidthBlock.class).instanceSize(); @@ -234,4 +235,28 @@ public class VariableWidthBlock } return validPositions; } + + @Override + public int filter(int[] positions, int positionCount, int[] matchedPositions, Function test) + { + int matchCount = 0; + for (int i = 0; i < positionCount; i++) { + if (valueIsNull != null && valueIsNull[positions[i] + arrayOffset]) { + continue; + } + + byte[] value = slice.slice(offsets[i + arrayOffset], offsets[i + arrayOffset + 1] - offsets[i + arrayOffset]).getBytes(); + if (test.apply(value)) { + matchedPositions[matchCount++] = positions[i]; + } + } + + return matchCount; + } + + @Override + public byte[] get(int position) + { + return slice.slice(offsets[position + arrayOffset], offsets[position + arrayOffset + 1] - offsets[position + arrayOffset]).getBytes(); + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlockBuilder.java b/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlockBuilder.java index cb6857423..64028d820 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlockBuilder.java +++ b/presto-spi/src/main/java/io/prestosql/spi/block/VariableWidthBlockBuilder.java @@ -41,9 +41,9 @@ import static io.prestosql.spi.block.BlockUtil.compactOffsets; import static io.prestosql.spi.block.BlockUtil.compactSlice; import static java.lang.Math.min; -public class VariableWidthBlockBuilder - extends AbstractVariableWidthBlock - implements BlockBuilder +public class VariableWidthBlockBuilder + extends AbstractVariableWidthBlock + implements BlockBuilder { private static final int INSTANCE_SIZE = ClassLayout.parseClass(VariableWidthBlockBuilder.class).instanceSize(); diff --git a/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorTableHandle.java b/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorTableHandle.java index 0694aadef..9d98fc039 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorTableHandle.java +++ b/presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorTableHandle.java @@ -13,8 +13,11 @@ */ package io.prestosql.spi.connector; +import io.prestosql.spi.predicate.Domain; import sun.reflect.generics.reflectiveObjects.NotImplementedException; +import java.util.function.Function; + public interface ConnectorTableHandle { default boolean isFilterSupported() @@ -90,4 +93,9 @@ public interface ConnectorTableHandle { return false; } + + default String getAdditionalFilterConditions(Function printer) + { + return ""; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/connector/Constraint.java b/presto-spi/src/main/java/io/prestosql/spi/connector/Constraint.java index 62b409b50..1eed9f16e 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/connector/Constraint.java +++ b/presto-spi/src/main/java/io/prestosql/spi/connector/Constraint.java @@ -24,7 +24,6 @@ import static java.util.Objects.requireNonNull; public class Constraint { - boolean isPushDownEnabled; //TODO: Rajeev: Temp,needs to be removed. private final TupleDomain summary; private final Optional>> predicate; @@ -55,7 +54,6 @@ public class Constraint this.summary = summary; this.predicate = predicate; - this.isPushDownEnabled = false; } public TupleDomain getSummary() @@ -67,14 +65,4 @@ public class Constraint { return predicate; } - - public boolean isPushDownEnabled() - { - return isPushDownEnabled; - } - - public void setPushDownEnabled(boolean pushDownEnabled) - { - isPushDownEnabled = pushDownEnabled; - } }