diff --git a/SparkOpOpenGauss/.idea/.gitignore b/SparkOpOpenGauss/.idea/.gitignore new file mode 100644 index 00000000..db193d80 --- /dev/null +++ b/SparkOpOpenGauss/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/../../../../:\gitRepo\SparkOpOG\.idea/dataSources/ +/dataSources.local.xml +# Zeppelin ignored files +/ZeppelinRemoteNotebooks/ +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/SparkOpOpenGauss/.idea/SparkOpOGForTest.iml b/SparkOpOpenGauss/.idea/SparkOpOGForTest.iml new file mode 100644 index 00000000..78b2cc53 --- /dev/null +++ b/SparkOpOpenGauss/.idea/SparkOpOGForTest.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/SparkOpOpenGauss.iml b/SparkOpOpenGauss/.idea/SparkOpOpenGauss.iml new file mode 100644 index 00000000..78b2cc53 --- /dev/null +++ b/SparkOpOpenGauss/.idea/SparkOpOpenGauss.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/codeStyles/Project.xml b/SparkOpOpenGauss/.idea/codeStyles/Project.xml new file mode 100644 index 00000000..919ce1f1 --- /dev/null +++ b/SparkOpOpenGauss/.idea/codeStyles/Project.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/codeStyles/codeStyleConfig.xml b/SparkOpOpenGauss/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 00000000..a55e7a17 --- /dev/null +++ b/SparkOpOpenGauss/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/compiler.xml b/SparkOpOpenGauss/.idea/compiler.xml new file mode 100644 index 00000000..c9bd0573 --- /dev/null +++ b/SparkOpOpenGauss/.idea/compiler.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/dataSources.xml b/SparkOpOpenGauss/.idea/dataSources.xml new file mode 100644 index 00000000..481af398 --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources.xml @@ -0,0 +1,11 @@ + + + + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://10.11.6.27:15432/postgres + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3.xml b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3.xml new file mode 100644 index 00000000..76bd8a63 --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3.xml @@ -0,0 +1,1338 @@ + + + + + 9.2.4 + mdy + true ACSST +true ACST +false ACT +true ADT +true AESST +false AEST +false AFT +true AKDT +false AKST +true ALMST +false ALMT +true AMST +false AMT +true ANAST +false ANAT +true ARST +false ART +false AST +true AWSST +false AWST +true AZOST +false AZOT +true AZST +false AZT +false Africa/Abidjan +false Africa/Accra +false Africa/Addis_Ababa +false Africa/Algiers +false Africa/Asmara +false Africa/Asmera +false Africa/Bamako +false Africa/Bangui +false Africa/Banjul +false Africa/Bissau +false Africa/Blantyre +false Africa/Brazzaville +false Africa/Bujumbura +false Africa/Cairo +false Africa/Casablanca +true Africa/Ceuta +false Africa/Conakry +false Africa/Dakar +false Africa/Dar_es_Salaam +false Africa/Djibouti +false Africa/Douala +false Africa/El_Aaiun +false Africa/Freetown +false Africa/Gaborone +false Africa/Harare +false Africa/Johannesburg +false Africa/Juba +false Africa/Kampala +false Africa/Khartoum +false Africa/Kigali +false Africa/Kinshasa +false Africa/Lagos +false Africa/Libreville +false Africa/Lome +false Africa/Luanda +false Africa/Lubumbashi +false Africa/Lusaka +false Africa/Malabo +false Africa/Maputo +false Africa/Maseru +false Africa/Mbabane +false Africa/Mogadishu +false Africa/Monrovia +false Africa/Nairobi +false Africa/Ndjamena +false Africa/Niamey +false Africa/Nouakchott +false Africa/Ouagadougou +false Africa/Porto-Novo +false Africa/Sao_Tome +false Africa/Timbuktu +false Africa/Tripoli +false Africa/Tunis +false Africa/Windhoek +true America/Adak +true America/Anchorage +false America/Anguilla +false America/Antigua +false America/Araguaina +false America/Argentina/Buenos_Aires +false America/Argentina/Catamarca +false America/Argentina/ComodRivadavia +false America/Argentina/Cordoba +false America/Argentina/Jujuy +false America/Argentina/La_Rioja +false America/Argentina/Mendoza +false America/Argentina/Rio_Gallegos +false America/Argentina/Salta +false America/Argentina/San_Juan +false America/Argentina/San_Luis +false America/Argentina/Tucuman +false America/Argentina/Ushuaia +false America/Aruba +false America/Asuncion +false America/Atikokan +true America/Atka +false America/Bahia +true America/Bahia_Banderas +false America/Barbados +false America/Belem +false America/Belize +false America/Blanc-Sablon +false America/Boa_Vista +false America/Bogota +true America/Boise +false America/Buenos_Aires +true America/Cambridge_Bay +false America/Campo_Grande +false America/Cancun +false America/Caracas +false America/Catamarca +false America/Cayenne +false America/Cayman +true America/Chicago +true America/Chihuahua +false America/Coral_Harbour +false America/Cordoba +false America/Costa_Rica +false America/Creston +false America/Cuiaba +false America/Curacao +false America/Danmarkshavn +false America/Dawson +false America/Dawson_Creek +true America/Denver +true America/Detroit +false America/Dominica +true America/Edmonton +false America/Eirunepe +false America/El_Salvador +true America/Ensenada +true America/Fort_Wayne +false America/Fortaleza +true America/Glace_Bay +true America/Godthab +true America/Goose_Bay +true America/Grand_Turk +false America/Grenada +false America/Guadeloupe +false America/Guatemala +false America/Guayaquil +false America/Guyana +true America/Halifax +true America/Havana +false America/Hermosillo +true America/Indiana/Indianapolis +true America/Indiana/Knox +true America/Indiana/Marengo +true America/Indiana/Petersburg +true America/Indiana/Tell_City +true America/Indiana/Vevay +true America/Indiana/Vincennes +true America/Indiana/Winamac +true America/Indianapolis +true America/Inuvik +true America/Iqaluit +false America/Jamaica +false America/Jujuy +true America/Juneau +true America/Kentucky/Louisville +true America/Kentucky/Monticello +true America/Knox_IN +false America/Kralendijk +false America/La_Paz +false America/Lima +true America/Los_Angeles +true America/Louisville +false America/Lower_Princes +false America/Maceio +false America/Managua +false America/Manaus +false America/Marigot +false America/Martinique +true America/Matamoros +true America/Mazatlan +false America/Mendoza +true America/Menominee +true America/Merida +true America/Metlakatla +true America/Mexico_City +true America/Miquelon +true America/Moncton +true America/Monterrey +false America/Montevideo +true America/Montreal +false America/Montserrat +true America/Nassau +true America/New_York +true America/Nipigon +true America/Nome +false America/Noronha +true America/North_Dakota/Beulah +true America/North_Dakota/Center +true America/North_Dakota/New_Salem +true America/Ojinaga +false America/Panama +true America/Pangnirtung +false America/Paramaribo +false America/Phoenix +true America/Port-au-Prince +false America/Port_of_Spain +false America/Porto_Acre +false America/Porto_Velho +false America/Puerto_Rico +true America/Rainy_River +true America/Rankin_Inlet +false America/Recife +false America/Regina +true America/Resolute +false America/Rio_Branco +false America/Rosario +true America/Santa_Isabel +false America/Santarem +true America/Santiago +false America/Santo_Domingo +false America/Sao_Paulo +true America/Scoresbysund +true America/Shiprock +true America/Sitka +false America/St_Barthelemy +true America/St_Johns +false America/St_Kitts +false America/St_Lucia +false America/St_Thomas +false America/St_Vincent +false America/Swift_Current +false America/Tegucigalpa +true America/Thule +true America/Thunder_Bay +true America/Tijuana +true America/Toronto +false America/Tortola +true America/Vancouver +false America/Virgin +false America/Whitehorse +true America/Winnipeg +true America/Yakutat +true America/Yellowknife +false Antarctica/Casey +false Antarctica/Davis +false Antarctica/DumontDUrville +false Antarctica/Macquarie +false Antarctica/Mawson +false Antarctica/McMurdo +false Antarctica/Palmer +false Antarctica/Rothera +false Antarctica/South_Pole +false Antarctica/Syowa +false Antarctica/Vostok +true Arctic/Longyearbyen +false Asia/Aden +false Asia/Almaty +true Asia/Amman +false Asia/Anadyr +false Asia/Aqtau +false Asia/Aqtobe +false Asia/Ashgabat +false Asia/Ashkhabad +false Asia/Baghdad +false Asia/Bahrain +false Asia/Baku +false Asia/Bangkok +false Asia/Beijing +true Asia/Beirut +false Asia/Bishkek +false Asia/Brunei +false Asia/Calcutta +false Asia/Choibalsan +false Asia/Chongqing +false Asia/Chungking +false Asia/Colombo +false Asia/Dacca +true Asia/Damascus +false Asia/Dhaka +false Asia/Dili +false Asia/Dubai +false Asia/Dushanbe +true Asia/Gaza +false Asia/Harbin +true Asia/Hebron +false Asia/Ho_Chi_Minh +false Asia/Hong_Kong +false Asia/Hovd +false Asia/Irkutsk +false Asia/Istanbul +false Asia/Jakarta +false Asia/Jayapura +true Asia/Jerusalem +false Asia/Kabul +false Asia/Kamchatka +false Asia/Karachi +false Asia/Kashgar +false Asia/Kathmandu +false Asia/Katmandu +false Asia/Khandyga +false Asia/Kolkata +false Asia/Krasnoyarsk +false Asia/Kuala_Lumpur +false Asia/Kuching +false Asia/Kuwait +false Asia/Macao +false Asia/Macau +false Asia/Magadan +false Asia/Makassar +false Asia/Manila +false Asia/Muscat +true Asia/Nicosia +false Asia/Novokuznetsk +false Asia/Novosibirsk +false Asia/Omsk +false Asia/Oral +false Asia/Phnom_Penh +false Asia/Pontianak +false Asia/Pyongyang +false Asia/Qatar +false Asia/Qyzylorda +false Asia/Rangoon +false Asia/Riyadh +false Asia/Saigon +false Asia/Sakhalin +false Asia/Samarkand +false Asia/Seoul +false Asia/Shanghai +false Asia/Singapore +false Asia/Taipei +false Asia/Tashkent +false Asia/Tbilisi +true Asia/Tehran +true Asia/Tel_Aviv +false Asia/Thimbu +false Asia/Thimphu +false Asia/Tokyo +false Asia/Ujung_Pandang +false Asia/Ulaanbaatar +false Asia/Ulan_Bator +false Asia/Urumqi +false Asia/Ust-Nera +false Asia/Vientiane +false Asia/Vladivostok +false Asia/Yakutsk +false Asia/Yekaterinburg +false Asia/Yerevan +true Atlantic/Azores +true Atlantic/Bermuda +true Atlantic/Canary +false Atlantic/Cape_Verde +true Atlantic/Faeroe +true Atlantic/Faroe +true Atlantic/Jan_Mayen +true Atlantic/Madeira +false Atlantic/Reykjavik +false Atlantic/South_Georgia +false Atlantic/St_Helena +false Atlantic/Stanley +false Australia/ACT +false Australia/Adelaide +false Australia/Brisbane +false Australia/Broken_Hill +false Australia/Canberra +false Australia/Currie +false Australia/Darwin +false Australia/Eucla +false Australia/Hobart +false Australia/LHI +false Australia/Lindeman +false Australia/Lord_Howe +false Australia/Melbourne +false Australia/NSW +false Australia/North +false Australia/Perth +false Australia/Queensland +false Australia/South +false Australia/Sydney +false Australia/Tasmania +false Australia/Victoria +false Australia/West +false Australia/Yancowinna +true BDST +false BDT +false BNT +false BORT +false BOT +false BRA +true BRST +false BRT +true BST +false BTT +false Brazil/Acre +false Brazil/DeNoronha +false Brazil/East +false Brazil/West +true CADT +false CAST +false CCT +true CDT +true CEST +true CET +true CETDST +true CHADT +false CHAST +false CHUT +false CKT +true CLST +false CLT +false COT +false CST +true CST6CDT +false CXT +true Canada/Atlantic +true Canada/Central +true Canada/Eastern +true Canada/Mountain +true Canada/Newfoundland +true Canada/Pacific +false Canada/Saskatchewan +false Canada/Yukon +true Chile/Continental +true Chile/EasterIsland +true Cuba +false DAVT +false DDUT +true EASST +false EAST +false EAT +true EDT +true EEST +false EET +true EETDST +true EGST +false EGT +false EST +true EST5EDT +false Egypt +false Eire +false Etc/GMT +false Etc/GMT+0 +false Etc/GMT+1 +false Etc/GMT+10 +false Etc/GMT+11 +false Etc/GMT+12 +false Etc/GMT+2 +false Etc/GMT+3 +false Etc/GMT+4 +false Etc/GMT+5 +false Etc/GMT+6 +false Etc/GMT+7 +false Etc/GMT+8 +false Etc/GMT+9 +false Etc/GMT-0 +false Etc/GMT-1 +false Etc/GMT-10 +false Etc/GMT-11 +false Etc/GMT-12 +false Etc/GMT-13 +false Etc/GMT-14 +false Etc/GMT-2 +false Etc/GMT-3 +false Etc/GMT-4 +false Etc/GMT-5 +false Etc/GMT-6 +false Etc/GMT-7 +false Etc/GMT-8 +false Etc/GMT-9 +false Etc/GMT0 +false Etc/Greenwich +false Etc/UCT +false Etc/UTC +false Etc/Universal +false Etc/Zulu +true Europe/Amsterdam +true Europe/Andorra +true Europe/Athens +true Europe/Belfast +true Europe/Belgrade +true Europe/Berlin +true Europe/Bratislava +true Europe/Brussels +true Europe/Bucharest +true Europe/Budapest +true Europe/Busingen +true Europe/Chisinau +true Europe/Copenhagen +false Europe/Dublin +true Europe/Gibraltar +true Europe/Guernsey +true Europe/Helsinki +true Europe/Isle_of_Man +false Europe/Istanbul +true Europe/Jersey +false Europe/Kaliningrad +true Europe/Kiev +true Europe/Lisbon +true Europe/Ljubljana +true Europe/London +true Europe/Luxembourg +true Europe/Madrid +true Europe/Malta +true Europe/Mariehamn +false Europe/Minsk +true Europe/Monaco +false Europe/Moscow +true Europe/Nicosia +true Europe/Oslo +true Europe/Paris +true Europe/Podgorica +true Europe/Prague +true Europe/Riga +true Europe/Rome +false Europe/Samara +true Europe/San_Marino +true Europe/Sarajevo +false Europe/Simferopol +true Europe/Skopje +true Europe/Sofia +true Europe/Stockholm +true Europe/Tallinn +true Europe/Tirane +true Europe/Tiraspol +true Europe/Uzhgorod +true Europe/Vaduz +true Europe/Vatican +true Europe/Vienna +true Europe/Vilnius +false Europe/Volgograd +true Europe/Warsaw +true Europe/Zagreb +true Europe/Zaporozhye +true Europe/Zurich +false FET +true FJST +false FJT +true FKST +false FKT +true FNST +false FNT +false GALT +false GAMT +true GB +true GB-Eire +true GEST +false GET +false GFT +false GILT +false GMT +false GMT+0 +false GMT-0 +false GMT0 +false GYT +false Greenwich +false HKT +false HST +false Hongkong +false ICT +true IDT +false IOT +true IRKST +false IRKT +false IRT +false IST +false Iceland +false Indian/Antananarivo +false Indian/Chagos +false Indian/Christmas +false Indian/Cocos +false Indian/Comoro +false Indian/Kerguelen +false Indian/Mahe +false Indian/Maldives +false Indian/Mauritius +false Indian/Mayotte +false Indian/Reunion +true Iran +true Israel +false JAYT +false JST +false Jamaica +false Japan +true KDT +true KGST +false KGT +false KOST +true KRAST +false KRAT +false KST +false Kwajalein +true LHDT +false LHST +false LIGT +false LINT +false LKT +false Libya +true MAGST +false MAGT +false MART +false MAWT +true MDT +true MEST +true MET +true METDST +false MEZ +false MHT +false MMT +false MPT +true MSD +false MSK +false MST +true MST7MDT +true MUST +false MUT +false MVT +false MYT +true Mexico/BajaNorte +true Mexico/BajaSur +true Mexico/General +true NDT +false NFT +true NOVST +false NOVT +false NPT +false NST +false NUT +false NZ +false NZ-CHAT +true NZDT +false NZST +false NZT +true Navajo +true OMSST +false OMST +true PDT +false PET +true PETST +false PETT +false PGT +false PHOT +false PHT +true PKST +false PKT +true PMDT +false PMST +false PONT +false PRC +false PST +true PST8PDT +false PWT +true PYST +false PYT +false Pacific/Apia +false Pacific/Auckland +false Pacific/Chatham +false Pacific/Chuuk +true Pacific/Easter +false Pacific/Efate +false Pacific/Enderbury +false Pacific/Fakaofo +false Pacific/Fiji +false Pacific/Funafuti +false Pacific/Galapagos +false Pacific/Gambier +false Pacific/Guadalcanal +false Pacific/Guam +false Pacific/Honolulu +false Pacific/Johnston +false Pacific/Kiritimati +false Pacific/Kosrae +false Pacific/Kwajalein +false Pacific/Majuro +false Pacific/Marquesas +false Pacific/Midway +false Pacific/Nauru +false Pacific/Niue +false Pacific/Norfolk +false Pacific/Noumea +false Pacific/Pago_Pago +false Pacific/Palau +false Pacific/Pitcairn +false Pacific/Pohnpei +false Pacific/Ponape +false Pacific/Port_Moresby +false Pacific/Rarotonga +false Pacific/Saipan +false Pacific/Samoa +false Pacific/Tahiti +false Pacific/Tarawa +false Pacific/Tongatapu +false Pacific/Truk +false Pacific/Wake +false Pacific/Wallis +false Pacific/Yap +true Poland +true Portugal +false RET +false ROC +false ROK +true SADT +false SAST +false SCT +false SGT +false Singapore +false TAHT +false TFT +false TJT +false TKT +false TMT +false TOT +false TRUT +false TVT +false Turkey +false UCT +true ULAST +false ULAT +true US/Alaska +true US/Aleutian +false US/Arizona +true US/Central +true US/East-Indiana +true US/Eastern +false US/Hawaii +true US/Indiana-Starke +true US/Michigan +true US/Mountain +true US/Pacific +true US/Pacific-New +false US/Samoa +false UT +false UTC +true UYST +false UYT +true UZST +false UZT +false Universal +false VET +true VLAST +false VLAT +false VOLT +false VUT +false W-SU +true WADT +false WAKT +false WAST +false WAT +true WDT +true WET +true WETDST +false WFT +true WGST +false WGT +true YAKST +false YAKT +false YAPT +true YEKST +false YEKT +false Z +false Zulu +true posixrules + + 1631545452 + + + 16384 + omm + + + 14178 + default administrative connection database + omm + 11789 + 1 + + + 32779 + omm + 11791 + + + 16385 + 1 + + + 10 + 1 + 1 + 1 + 1 + 1 + + + 16389 + 1 + + + 100 + reserved schema for DELTA tables + 1 + omm + + + 4988 + dbe_perf schema + 1 + omm + + + 16387 + 9028 + gaussdb + + + 13325 + 2 + omm + + + 11 + system catalog schema + 2 + omm + + + 3988 + pkg_service schema + 1 + omm + + + 2200 + standard public schema + 2 + omm + + + 4989 + snapshot schema + 1 + omm + + + 16391 + 9315 + sparkuser + 11790 + 1 + + + 13619 + 2 + omm + hdfs_fdw_handler + pg_catalog + hdfs_fdw_validator + pg_catalog + + + 13608 + 2 + omm + dist_fdw_handler + pg_catalog + dist_fdw_validator + pg_catalog + + + 13613 + 2 + omm + file_fdw_handler + pg_catalog + file_fdw_validator + pg_catalog + + + 13618 + 2 + omm + hdfs_fdw_handler + pg_catalog + hdfs_fdw_validator + pg_catalog + + + 13623 + 2 + omm + log_fdw_handler + pg_catalog + log_fdw_validator + pg_catalog + + + 13665 + 2 + omm + mot_fdw_handler + pg_catalog + mot_fdw_validator + pg_catalog + + + 403 + b-tree index access method + 1 + + + 405 + hash index access method + 1 + + + 783 + GiST index access method + 1 + + + 2742 + GIN index access method + 1 + + + 4000 + SP-GiST index access method + 1 + + + 4039 + psort index access method + 1 + + + 4239 + cstore btree index access method + 1 + + + 4444 + cstore GIN index access method + 1 + + + 13605 + foreign-data wrapper for distfs access + 2 + 1.0 + pg_catalog + 11 + + + 13610 + foreign-data wrapper for flat file access + 2 + 1.0 + pg_catalog + 11 + + + 13615 + foreign-data wrapper for flat file access + 2 + 1.0 + pg_catalog + 11 + + + 13626 + data type for storing sets of (key, value) pairs + 2 + 1.1 + pg_catalog + 11 + + + 13620 + Foreign Data Wrapper for accessing logging data + 2 + 1.0 + pg_catalog + 11 + + + 13662 + foreign-data wrapper for MOT access + 2 + 1.0 + pg_catalog + 11 + + + 11746 + PL/pgSQL procedural language + 2 + 1.0 + pg_catalog + 11 + + + 13667 + provides security functionality + 2 + 1.0 + pg_catalog + 11 + + + + + + + + 100 + reserved schema for DELTA tables + 1 + omm + + + 4988 + dbe_perf schema + 1 + omm + + + 13325 + 2 + omm + + + 11 + system catalog schema + 2 + omm + + + 3988 + pkg_service schema + 1 + omm + + + 2200 + standard public schema + 2 + omm + 1 + + + 4989 + snapshot schema + 1 + omm + + + 13619 + 2 + omm + hdfs_fdw_handler + pg_catalog + hdfs_fdw_validator + pg_catalog + + + 13608 + 2 + omm + dist_fdw_handler + pg_catalog + dist_fdw_validator + pg_catalog + + + 13613 + 2 + omm + file_fdw_handler + pg_catalog + file_fdw_validator + pg_catalog + + + 13618 + 2 + omm + hdfs_fdw_handler + pg_catalog + hdfs_fdw_validator + pg_catalog + + + 13623 + 2 + omm + log_fdw_handler + pg_catalog + log_fdw_validator + pg_catalog + + + 13665 + 2 + omm + mot_fdw_handler + pg_catalog + mot_fdw_validator + pg_catalog + + + 403 + b-tree index access method + 1 + + + 405 + hash index access method + 1 + + + 783 + GiST index access method + 1 + + + 2742 + GIN index access method + 1 + + + 4000 + SP-GiST index access method + 1 + + + 4039 + psort index access method + 1 + + + 4239 + cstore btree index access method + 1 + + + 4444 + cstore GIN index access method + 1 + + + 13605 + foreign-data wrapper for distfs access + 2 + 1.0 + pg_catalog + 11 + + + 13610 + foreign-data wrapper for flat file access + 2 + 1.0 + pg_catalog + 11 + + + 13615 + foreign-data wrapper for flat file access + 2 + 1.0 + pg_catalog + 11 + + + 13626 + data type for storing sets of (key, value) pairs + 2 + 1.1 + pg_catalog + 11 + + + 13620 + Foreign Data Wrapper for accessing logging data + 2 + 1.0 + pg_catalog + 11 + + + 13662 + foreign-data wrapper for MOT access + 2 + 1.0 + pg_catalog + 11 + + + 11746 + PL/pgSQL procedural language + 2 + 1.0 + pg_catalog + 11 + + + 13667 + provides security functionality + 2 + 1.0 + pg_catalog + 11 + + + + + + + + 24587 + sparkuser + 9908 +
+ + 24581 + sparkuser + 9612 +
+ + 13609 + 2 + omm + + + 13614 + 2 + omm + + + 13624 + 2 + omm + + + 13666 + 2 + omm + + + 13609 + 2 + omm + + + 13614 + 2 + omm + + + 13624 + 2 + omm + + + 13666 + 2 + omm + + + 1 + integer|0s + 1 + 9908 + 23 + + + 2 + text|0s + 1 + 9908 + 25 + + + 3 + integer|0s + 1 + 9908 + 23 + + + 4 + char(50)|0s + 9908 + 1042 + + + 5 + real|0s + 9908 + 700 + + + 24593 + 9908 + id + 1 + 1 + + + 24594 + 9908 + id + 1 + company1_pkey + + + 1 + integer|0s + 9612 + 23 + + + 2 + text|0s + 9612 + 25 + + + 3 + numeric|0s + 9612 + 1700 + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ.meta new file mode 100644 index 00000000..5b992090 --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ.meta @@ -0,0 +1 @@ +#n:postgres \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/information_schema.FNRwLQ.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/information_schema.FNRwLQ.meta new file mode 100644 index 00000000..1ff3db2e --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/information_schema.FNRwLQ.meta @@ -0,0 +1,2 @@ +#n:information_schema +! [null, 0, null, null, -2147483648, -2147483648] diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/pg_catalog.0S1ZNQ.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/pg_catalog.0S1ZNQ.meta new file mode 100644 index 00000000..44e65b16 --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/pg_catalog.0S1ZNQ.meta @@ -0,0 +1,2 @@ +#n:pg_catalog +! [null, 0, null, null, -2147483648, -2147483648] diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/sparkuser.KCgh4Q.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/sparkuser.KCgh4Q.meta new file mode 100644 index 00000000..c1997422 --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/sparkuser.KCgh4Q.meta @@ -0,0 +1,2 @@ +#n:sparkuser +! [11790, 0, null, null, -2147483648, -2147483648] diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ.meta new file mode 100644 index 00000000..d35b334d --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ.meta @@ -0,0 +1 @@ +#n:school \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/information_schema.FNRwLQ.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/information_schema.FNRwLQ.meta new file mode 100644 index 00000000..1ff3db2e --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/information_schema.FNRwLQ.meta @@ -0,0 +1,2 @@ +#n:information_schema +! [null, 0, null, null, -2147483648, -2147483648] diff --git a/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/pg_catalog.0S1ZNQ.meta b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/pg_catalog.0S1ZNQ.meta new file mode 100644 index 00000000..44e65b16 --- /dev/null +++ b/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/pg_catalog.0S1ZNQ.meta @@ -0,0 +1,2 @@ +#n:pg_catalog +! [null, 0, null, null, -2147483648, -2147483648] diff --git a/SparkOpOpenGauss/.idea/google-java-format.xml b/SparkOpOpenGauss/.idea/google-java-format.xml new file mode 100644 index 00000000..2aa056da --- /dev/null +++ b/SparkOpOpenGauss/.idea/google-java-format.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/jarRepositories.xml b/SparkOpOpenGauss/.idea/jarRepositories.xml new file mode 100644 index 00000000..329b2f85 --- /dev/null +++ b/SparkOpOpenGauss/.idea/jarRepositories.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/libraries/opengauss_jdbc_2_0_0.xml b/SparkOpOpenGauss/.idea/libraries/opengauss_jdbc_2_0_0.xml new file mode 100644 index 00000000..13fbcc3e --- /dev/null +++ b/SparkOpOpenGauss/.idea/libraries/opengauss_jdbc_2_0_0.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/misc.xml b/SparkOpOpenGauss/.idea/misc.xml new file mode 100644 index 00000000..4b661a5f --- /dev/null +++ b/SparkOpOpenGauss/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/uiDesigner.xml b/SparkOpOpenGauss/.idea/uiDesigner.xml new file mode 100644 index 00000000..e96534fb --- /dev/null +++ b/SparkOpOpenGauss/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/.idea/vcs.xml b/SparkOpOpenGauss/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/SparkOpOpenGauss/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/README.md b/SparkOpOpenGauss/README.md new file mode 100644 index 00000000..f29ae176 --- /dev/null +++ b/SparkOpOpenGauss/README.md @@ -0,0 +1,18 @@ +# SparkOpOpenGauss + +这是将 openGuass 作为数据源的spark示例代码。 + +**环境和软件版本要求（前置要求）：** + +1. Java 1.8 +2. Scala 2.12 +3. postgresql.jar 或 opengauss-jdbc--${version}.jar(自己打包或官方提供的jar包) + +**说明：** + +1. 请确保服务器上的数据库正常运行，且你的机器可正常连接数据库 +2. 以sparkuser用户身份执行./resources/school.sql文件 +3. 请修改代码中连接数据库的ip及端口，即修改 x.x.x.x:port 。主要代码内容在 ./src/main/scala/ +4. 可以在idea运行本示例。 + + 可运行./src/test/scala/org/opengauss/spark/OpenGaussExample。 + + 另一个直接使用Spark JDBC的例子是 ./src/main/scala/SQLDataSourceExample。 \ No newline at end of file diff --git a/SparkOpOpenGauss/SparkOpOG.iml b/SparkOpOpenGauss/SparkOpOG.iml new file mode 100644 index 00000000..78b2cc53 --- /dev/null +++ b/SparkOpOpenGauss/SparkOpOG.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/SparkOpOpenGauss/libs/opengauss-jdbc-2.0.0.jar b/SparkOpOpenGauss/libs/opengauss-jdbc-2.0.0.jar new file mode 100644 index 00000000..12320961 Binary files /dev/null and b/SparkOpOpenGauss/libs/opengauss-jdbc-2.0.0.jar differ diff --git a/SparkOpOpenGauss/pom.xml b/SparkOpOpenGauss/pom.xml new file mode 100644 index 00000000..16511042 --- /dev/null +++ b/SparkOpOpenGauss/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + org.example + SparkOpOG + 3.3.0-SNAPSHOT + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + WDF TestSuite.txt + + + + test + + test + + + + + + + + + + + + org.scalatest + scalatest_2.12 + 3.0.0 + test + + + org.apache.spark + spark-core_2.12 + 3.3.0-SNAPSHOT + + + + org.apache.spark + spark-sql_2.12 + 3.3.0-SNAPSHOT + + + + + + + + + + + + + + + + + + + + + + + + + Apache + https://repository.apache.org/snapshots/ + + true + + + + \ No newline at end of file diff --git a/SparkOpOpenGauss/resources/school.sql b/SparkOpOpenGauss/resources/school.sql new file mode 100644 index 00000000..f830b05d --- /dev/null +++ b/SparkOpOpenGauss/resources/school.sql @@ -0,0 +1,214 @@ +create database school; + +\c school; + +BEGIN; + +-- 创建表student +CREATE TABLE student +( + std_id INT PRIMARY KEY, + std_name VARCHAR(20) NOT NULL, + std_sex VARCHAR(6), + std_birth DATE, + std_in DATE NOT NULL, + std_address VARCHAR(100) +); + +-- 创建表teacher +CREATE TABLE teacher +( + tec_id INT PRIMARY KEY, + tec_name VARCHAR(20) NOT NULL, + tec_job VARCHAR(15), + tec_sex VARCHAR(6), + tec_age INT, + tec_in DATE NOT NULL +); + +-- 创建表class +CREATE TABLE class +( + cla_id INT PRIMARY KEY, + cla_name VARCHAR(20) NOT NULL, + cla_teacher INT NOT NULL +); +-- 给表class添加外键约束 +ALTER TABLE class ADD CONSTRAINT fk_tec_id FOREIGN KEY (cla_teacher) REFERENCES teacher(tec_id) ON DELETE CASCADE; + +-- 创建表school_department +CREATE TABLE school_department +( + depart_id INT PRIMARY KEY, + depart_name VARCHAR(30) NOT NULL, + depart_teacher INT NOT NULL +); +-- 给表school_department添加外键约束 +ALTER TABLE school_department ADD CONSTRAINT fk_depart_tec_id FOREIGN KEY (depart_teacher) REFERENCES teacher(tec_id) ON DELETE CASCADE; + +-- 创建表course +CREATE TABLE course +( + cor_id INT PRIMARY KEY, + cor_name VARCHAR(30) NOT NULL, + cor_type VARCHAR(20), + credit DOUBLE PRECISION +); + +-- 插入数据 +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (1,'张一','男','1993-01-01','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (2,'张二','男','1993-01-02','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (3,'张三','男','1993-01-03','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (4,'张四','男','1993-01-04','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (5,'张五','男','1993-01-05','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (6,'张六','男','1993-01-06','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (7,'张七','男','1993-01-07','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (8,'张八','男','1993-01-08','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (9,'张九','男','1993-01-09','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (10,'李一','男','1993-01-10','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (11,'李二','男','1993-01-11','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (12,'李三','男','1993-01-12','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (13,'李四','男','1993-01-13','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (14,'李五','男','1993-01-14','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (15,'李六','男','1993-01-15','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (16,'李七','男','1993-01-16','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (17,'李八','男','1993-01-17','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (18,'李九','男','1993-01-18','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (19,'王一','男','1993-01-19','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (20,'王二','男','1993-01-20','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (21,'王三','男','1993-01-21','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (22,'王四','男','1993-01-22','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (23,'王五','男','1993-01-23','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (24,'王六','男','1993-01-24','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (25,'王七','男','1993-01-25','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (26,'王八','男','1993-01-26','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (27,'王九','男','1993-01-27','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (28,'钱一','男','1993-01-28','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (29,'钱二','男','1993-01-29','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (30,'钱三','男','1993-01-30','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (31,'钱四','男','1993-02-01','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (32,'钱五','男','1993-02-02','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (33,'钱六','男','1993-02-03','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (34,'钱七','男','1993-02-04','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (35,'钱八','男','1993-02-05','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (36,'钱九','男','1993-02-06','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (37,'吴一','男','1993-02-07','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (38,'吴二','男','1993-02-08','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (39,'吴三','男','1993-02-09','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (40,'吴四','男','1993-02-10','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (41,'吴五','男','1993-02-11','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (42,'吴六','男','1993-02-12','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (43,'吴七','男','1993-02-13','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (44,'吴八','男','1993-02-14','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (45,'吴九','男','1993-02-15','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (46,'柳一','男','1993-02-16','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (47,'柳二','男','1993-02-17','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (48,'柳三','男','1993-02-18','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (49,'柳四','男','1993-02-19','2011-09-01','江苏省南京市雨花台区'); +INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (50,'柳五','男','1993-02-20','2011-09-01','江苏省南京市雨花台区'); + +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (1,'张一','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (2,'张二','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (3,'张三','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (4,'张四','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (5,'张五','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (6,'张六','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (7,'张七','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (8,'张八','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (9,'张九','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (10,'李一','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (11,'李二','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (12,'李三','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (13,'李四','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (14,'李五','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (15,'李六','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (16,'李七','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (17,'李八','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (18,'李九','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (19,'王一','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (20,'王二','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (21,'王三','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (22,'王四','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (23,'王五','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (24,'王六','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (25,'王七','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (26,'王八','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (27,'王九','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (28,'钱一','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (29,'钱二','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (30,'钱三','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (31,'钱四','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (32,'钱五','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (33,'钱六','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (34,'钱七','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (35,'钱八','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (36,'钱九','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (37,'吴一','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (38,'吴二','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (39,'吴三','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (40,'吴四','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (41,'吴五','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (42,'吴六','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (43,'吴七','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (44,'吴八','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (45,'吴九','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (46,'柳一','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (47,'柳二','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (48,'柳三','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (49,'柳四','讲师','男',35,'2009-07-01'); +INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (50,'柳五','讲师','男',35,'2009-07-01'); + +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (1,'计算机',1); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (2,'自动化',3); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (3,'飞行器设计',5); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (4,'大学物理',7); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (5,'高等数学',9); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (6,'大学化学',12); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (7,'表演',14); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (8,'服装设计',16); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (9,'工业设计',18); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (10,'金融学',21); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (11,'医学',23); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (12,'土木工程',25); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (13,'机械',27); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (14,'建筑学',29); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (15,'经济学',32); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (16,'财务管理',34); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (17,'人力资源',36); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (18,'力学',38); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (19,'人工智能',41); +INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (20,'会计',45); + +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (1,'计算机学院',2); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (2,'自动化学院',4); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (3,'航空宇航学院',6); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (4,'艺术学院',8); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (5,'理学院',11); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (6,'人工智能学院',13); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (7,'工学院',15); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (8,'管理学院',17); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (9,'农学院',22); +INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (10,'医学院',28); + +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (1,'数据库系统概论','必修',3); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (2,'艺术设计概论','选修',1); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (3,'力学制图','必修',4); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (4,'飞行器设计历史','选修',1); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (5,'马克思主义','必修',2); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (6,'大学历史','必修',2); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (7,'人力资源管理理论','必修',2.5); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (8,'线性代数','必修',4); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (9,'JAVA程序设计','必修',3); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (10,'操作系统','必修',4); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (11,'计算机组成原理','必修',3); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (12,'自动化设计理论','必修',2); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (13,'情绪表演','必修',2.5); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (14,'茶学历史','选修',1); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (15,'艺术论','必修',1.5); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (16,'机器学习','必修',3); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (17,'数据挖掘','选修',2); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (18,'图像识别','必修',3); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (19,'解剖学','必修',4); +INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (20,'3D max','选修',2); + +COMMIT; \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/resources/adult.csv b/SparkOpOpenGauss/src/main/resources/adult.csv new file mode 100644 index 00000000..2aae7c3d --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/adult.csv @@ -0,0 +1,102 @@ +age,workclass,fnlwgt,education,education_num,martial_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,salary +39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White, Male, 2174, 0, 40, United-States, <=50K +50, Self-emp-not-inc, 83311, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 13, United-States, <=50K +38, Private, 215646, HS-grad, 9, Divorced, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K +53, Private, 234721, 11th, 7, Married-civ-spouse, Handlers-cleaners, Husband, Black, Male, 0, 0, 40, United-States, <=50K +28, Private, 338409, Bachelors, 13, Married-civ-spouse, Prof-specialty, Wife, Black, Female, 0, 0, 40, Cuba, <=50K +37, Private, 284582, Masters, 14, Married-civ-spouse, Exec-managerial, Wife, White, Female, 0, 0, 40, United-States, <=50K +49, Private, 160187, 9th, 5, Married-spouse-absent, Other-service, Not-in-family, Black, Female, 0, 0, 16, Jamaica, <=50K +52, Self-emp-not-inc, 209642, HS-grad, 9, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 45, United-States, >50K +31, Private, 45781, Masters, 14, Never-married, Prof-specialty, Not-in-family, White, Female, 14084, 0, 50, United-States, >50K +42, Private, 159449, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 5178, 0, 40, United-States, >50K +37, Private, 280464, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, Black, Male, 0, 0, 80, United-States, >50K +30, State-gov, 141297, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Asian-Pac-Islander, Male, 0, 0, 40, India, >50K +23, Private, 122272, Bachelors, 13, Never-married, Adm-clerical, Own-child, White, Female, 0, 0, 30, United-States, <=50K +32, Private, 205019, Assoc-acdm, 12, Never-married, Sales, Not-in-family, Black, Male, 0, 0, 50, United-States, <=50K +40, Private, 121772, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male, 0, 0, 40, ?, >50K +34, Private, 245487, 7th-8th, 4, Married-civ-spouse, Transport-moving, Husband, Amer-Indian-Eskimo, Male, 0, 0, 45, Mexico, <=50K +25, Self-emp-not-inc, 176756, HS-grad, 9, Never-married, Farming-fishing, Own-child, White, Male, 0, 0, 35, United-States, <=50K +32, Private, 186824, HS-grad, 9, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, United-States, <=50K +38, Private, 28887, 11th, 7, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 50, United-States, <=50K +43, Self-emp-not-inc, 292175, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 45, United-States, >50K +40, Private, 193524, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 60, United-States, >50K +54, Private, 302146, HS-grad, 9, Separated, Other-service, Unmarried, Black, Female, 0, 0, 20, United-States, <=50K +35, Federal-gov, 76845, 9th, 5, Married-civ-spouse, Farming-fishing, Husband, Black, Male, 0, 0, 40, United-States, <=50K +43, Private, 117037, 11th, 7, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 2042, 40, United-States, <=50K +59, Private, 109015, HS-grad, 9, Divorced, Tech-support, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +56, Local-gov, 216851, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K +19, Private, 168294, HS-grad, 9, Never-married, Craft-repair, Own-child, White, Male, 0, 0, 40, United-States, <=50K +54, ?, 180211, Some-college, 10, Married-civ-spouse, ?, Husband, Asian-Pac-Islander, Male, 0, 0, 60, South, >50K +39, Private, 367260, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 80, United-States, <=50K +49, Private, 193366, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K +23, Local-gov, 190709, Assoc-acdm, 12, Never-married, Protective-serv, Not-in-family, White, Male, 0, 0, 52, United-States, <=50K +20, Private, 266015, Some-college, 10, Never-married, Sales, Own-child, Black, Male, 0, 0, 44, United-States, <=50K +45, Private, 386940, Bachelors, 13, Divorced, Exec-managerial, Own-child, White, Male, 0, 1408, 40, United-States, <=50K +30, Federal-gov, 59951, Some-college, 10, Married-civ-spouse, Adm-clerical, Own-child, White, Male, 0, 0, 40, United-States, <=50K +22, State-gov, 311512, Some-college, 10, Married-civ-spouse, Other-service, Husband, Black, Male, 0, 0, 15, United-States, <=50K +48, Private, 242406, 11th, 7, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, Puerto-Rico, <=50K +21, Private, 197200, Some-college, 10, Never-married, Machine-op-inspct, Own-child, White, Male, 0, 0, 40, United-States, <=50K +19, Private, 544091, HS-grad, 9, Married-AF-spouse, Adm-clerical, Wife, White, Female, 0, 0, 25, United-States, <=50K +31, Private, 84154, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 38, ?, >50K +48, Self-emp-not-inc, 265477, Assoc-acdm, 12, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K +31, Private, 507875, 9th, 5, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 43, United-States, <=50K +53, Self-emp-not-inc, 88506, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K +24, Private, 172987, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 50, United-States, <=50K +49, Private, 94638, HS-grad, 9, Separated, Adm-clerical, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +25, Private, 289980, HS-grad, 9, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 35, United-States, <=50K +57, Federal-gov, 337895, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Black, Male, 0, 0, 40, United-States, >50K +53, Private, 144361, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 38, United-States, <=50K +44, Private, 128354, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +41, State-gov, 101603, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K +29, Private, 271466, Assoc-voc, 11, Never-married, Prof-specialty, Not-in-family, White, Male, 0, 0, 43, United-States, <=50K +25, Private, 32275, Some-college, 10, Married-civ-spouse, Exec-managerial, Wife, Other, Female, 0, 0, 40, United-States, <=50K +18, Private, 226956, HS-grad, 9, Never-married, Other-service, Own-child, White, Female, 0, 0, 30, ?, <=50K +47, Private, 51835, Prof-school, 15, Married-civ-spouse, Prof-specialty, Wife, White, Female, 0, 1902, 60, Honduras, >50K +50, Federal-gov, 251585, Bachelors, 13, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 55, United-States, >50K +47, Self-emp-inc, 109832, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 60, United-States, <=50K +43, Private, 237993, Some-college, 10, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K +46, Private, 216666, 5th-6th, 3, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 40, Mexico, <=50K +35, Private, 56352, Assoc-voc, 11, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Puerto-Rico, <=50K +41, Private, 147372, HS-grad, 9, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 48, United-States, <=50K +30, Private, 188146, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 5013, 0, 40, United-States, <=50K +30, Private, 59496, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 2407, 0, 40, United-States, <=50K +32, ?, 293936, 7th-8th, 4, Married-spouse-absent, ?, Not-in-family, White, Male, 0, 0, 40, ?, <=50K +48, Private, 149640, HS-grad, 9, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 0, 40, United-States, <=50K +42, Private, 116632, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 45, United-States, >50K +29, Private, 105598, Some-college, 10, Divorced, Tech-support, Not-in-family, White, Male, 0, 0, 58, United-States, <=50K +36, Private, 155537, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K +28, Private, 183175, Some-college, 10, Divorced, Adm-clerical, Not-in-family, White, Female, 0, 0, 40, United-States, <=50K +53, Private, 169846, HS-grad, 9, Married-civ-spouse, Adm-clerical, Wife, White, Female, 0, 0, 40, United-States, >50K +49, Self-emp-inc, 191681, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 50, United-States, >50K +25, ?, 200681, Some-college, 10, Never-married, ?, Own-child, White, Male, 0, 0, 40, United-States, <=50K +19, Private, 101509, Some-college, 10, Never-married, Prof-specialty, Own-child, White, Male, 0, 0, 32, United-States, <=50K +31, Private, 309974, Bachelors, 13, Separated, Sales, Own-child, Black, Female, 0, 0, 40, United-States, <=50K +29, Self-emp-not-inc, 162298, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 70, United-States, >50K +23, Private, 211678, Some-college, 10, Never-married, Machine-op-inspct, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K +79, Private, 124744, Some-college, 10, Married-civ-spouse, Prof-specialty, Other-relative, White, Male, 0, 0, 20, United-States, <=50K +27, Private, 213921, HS-grad, 9, Never-married, Other-service, Own-child, White, Male, 0, 0, 40, Mexico, <=50K +40, Private, 32214, Assoc-acdm, 12, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 40, United-States, <=50K +67, ?, 212759, 10th, 6, Married-civ-spouse, ?, Husband, White, Male, 0, 0, 2, United-States, <=50K +18, Private, 309634, 11th, 7, Never-married, Other-service, Own-child, White, Female, 0, 0, 22, United-States, <=50K +31, Local-gov, 125927, 7th-8th, 4, Married-civ-spouse, Farming-fishing, Husband, White, Male, 0, 0, 40, United-States, <=50K +18, Private, 446839, HS-grad, 9, Never-married, Sales, Not-in-family, White, Male, 0, 0, 30, United-States, <=50K +52, Private, 276515, Bachelors, 13, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Cuba, <=50K +46, Private, 51618, HS-grad, 9, Married-civ-spouse, Other-service, Wife, White, Female, 0, 0, 40, United-States, <=50K +59, Private, 159937, HS-grad, 9, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, <=50K +44, Private, 343591, HS-grad, 9, Divorced, Craft-repair, Not-in-family, White, Female, 14344, 0, 40, United-States, >50K +53, Private, 346253, HS-grad, 9, Divorced, Sales, Own-child, White, Female, 0, 0, 35, United-States, <=50K +49, Local-gov, 268234, HS-grad, 9, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K +33, Private, 202051, Masters, 14, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 50, United-States, <=50K +30, Private, 54334, 9th, 5, Never-married, Sales, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K +43, Federal-gov, 410867, Doctorate, 16, Never-married, Prof-specialty, Not-in-family, White, Female, 0, 0, 50, United-States, >50K +57, Private, 249977, Assoc-voc, 11, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K +37, Private, 286730, Some-college, 10, Divorced, Craft-repair, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +28, Private, 212563, Some-college, 10, Divorced, Machine-op-inspct, Unmarried, Black, Female, 0, 0, 25, United-States, <=50K +30, Private, 117747, HS-grad, 9, Married-civ-spouse, Sales, Wife, Asian-Pac-Islander, Female, 0, 1573, 35, ?, <=50K +34, Local-gov, 226296, Bachelors, 13, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K +29, Local-gov, 115585, Some-college, 10, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 50, United-States, <=50K +48, Self-emp-not-inc, 191277, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 1902, 60, United-States, >50K +37, Private, 202683, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, >50K +48, Private, 171095, Assoc-acdm, 12, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, England, <=50K +32, Federal-gov, 249409, HS-grad, 9, Never-married, Other-service, Own-child, Black, Male, 0, 0, 40, United-States, <=50K +76, Private, 124191, Masters, 14, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 40, United-States, >50K diff --git a/SparkOpOpenGauss/src/main/resources/customers.csv b/SparkOpOpenGauss/src/main/resources/customers.csv new file mode 100644 index 00000000..0b4f8528 --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/customers.csv @@ -0,0 +1,5 @@ +customerId,customerName +1,John +2,Clerk +3,Micheal +4,Sample diff --git a/SparkOpOpenGauss/src/main/resources/metrics.properties b/SparkOpOpenGauss/src/main/resources/metrics.properties new file mode 100644 index 00000000..a882870e --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/metrics.properties @@ -0,0 +1 @@ +*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/resources/multicharacterseperator.csv b/SparkOpOpenGauss/src/main/resources/multicharacterseperator.csv new file mode 100644 index 00000000..9db32e5b --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/multicharacterseperator.csv @@ -0,0 +1,3 @@ +a||b||c||d +1||2||3||4 +5||6||7||8 \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/resources/nested/a.csv b/SparkOpOpenGauss/src/main/resources/nested/a.csv new file mode 100644 index 00000000..9db32e5b --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/nested/a.csv @@ -0,0 +1,3 @@ +a||b||c||d +1||2||3||4 +5||6||7||8 \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/resources/nested/folder1/b.csv b/SparkOpOpenGauss/src/main/resources/nested/folder1/b.csv new file mode 100644 index 00000000..9db32e5b --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/nested/folder1/b.csv @@ -0,0 +1,3 @@ +a||b||c||d +1||2||3||4 +5||6||7||8 \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/resources/sales.csv b/SparkOpOpenGauss/src/main/resources/sales.csv new file mode 100644 index 00000000..b3145638 --- /dev/null +++ b/SparkOpOpenGauss/src/main/resources/sales.csv @@ -0,0 +1,15 @@ +transactionId,customerId,itemId,amountPaid +111,1,1,100.0 +112,2,2,505.0 +113,3,3,510.0 +114,4,4,600.0 +115,1,2,500.0 +116,1,2,500.0 +117,1,2,500.0 +118,1,2,500.0 +119,2,3,500.0 +120,1,2,500.0 +121,1,4,500.0 +122,1,2,500.0 +123,1,4,500.0 +124,1,2,500.0 \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/scala/SQLDataSourceExample.scala b/SparkOpOpenGauss/src/main/scala/SQLDataSourceExample.scala new file mode 100644 index 00000000..440851c3 --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/SQLDataSourceExample.scala @@ -0,0 +1,400 @@ +package org.apache.spark.examples.sql + +import java.util.Properties + +import org.apache.spark.sql.SparkSession + +object SQLDataSourceExample { + + case class Person(name: String, age: Long) + + def main(args: Array[String]): Unit = { + val spark = SparkSession + .builder() + .master("local") + .appName("Spark SQL data sources example") + .config("spark.some.config.option", "some-value") + .getOrCreate() + + // runBasicDataSourceExample(spark) + // runGenericFileSourceOptionsExample(spark) + // runBasicParquetExample(spark) + // runParquetSchemaMergingExample(spark) + // runJsonDatasetExample(spark) + // runCsvDatasetExample(spark) + // runTextDatasetExample(spark) + runJdbcDatasetExample(spark) + + spark.stop() + } + + private def runGenericFileSourceOptionsExample(spark: SparkSession): Unit = { + // $example on:ignore_corrupt_files$ + // enable ignore corrupt files + spark.sql("set spark.sql.files.ignoreCorruptFiles=true") + // dir1/file3.json is corrupt from parquet's view + val testCorruptDF = spark.read.parquet( + "examples/src/main/resources/dir1/", + "examples/src/main/resources/dir1/dir2/") + testCorruptDF.show() + // +-------------+ + // | file| + // +-------------+ + // |file1.parquet| + // |file2.parquet| + // +-------------+ + // $example off:ignore_corrupt_files$ + // $example on:recursive_file_lookup$ + val recursiveLoadedDF = spark.read.format("parquet") + .option("recursiveFileLookup", "true") + .load("examples/src/main/resources/dir1") + recursiveLoadedDF.show() + // +-------------+ + // | file| + // +-------------+ + // |file1.parquet| + // |file2.parquet| + // +-------------+ + // $example off:recursive_file_lookup$ + spark.sql("set spark.sql.files.ignoreCorruptFiles=false") + // $example on:load_with_path_glob_filter$ + val testGlobFilterDF = spark.read.format("parquet") + .option("pathGlobFilter", "*.parquet") // json file should be filtered out + .load("examples/src/main/resources/dir1") + testGlobFilterDF.show() + // +-------------+ + // | file| + // +-------------+ + // |file1.parquet| + // +-------------+ + // $example off:load_with_path_glob_filter$ + // $example on:load_with_modified_time_filter$ + val beforeFilterDF = spark.read.format("parquet") + // Files modified before 07/01/2020 at 05:30 are allowed + .option("modifiedBefore", "2020-07-01T05:30:00") + .load("examples/src/main/resources/dir1"); + beforeFilterDF.show(); + // +-------------+ + // | file| + // +-------------+ + // |file1.parquet| + // +-------------+ + val afterFilterDF = spark.read.format("parquet") + // Files modified after 06/01/2020 at 05:30 are allowed + .option("modifiedAfter", "2020-06-01T05:30:00") + .load("examples/src/main/resources/dir1"); + afterFilterDF.show(); + // +-------------+ + // | file| + // +-------------+ + // +-------------+ + // $example off:load_with_modified_time_filter$ + } + + private def runBasicDataSourceExample(spark: SparkSession): Unit = { + // $example on:generic_load_save_functions$ + val usersDF = spark.read.load("examples/src/main/resources/users.parquet") + usersDF.select("name", "favorite_color").write.save("namesAndFavColors.parquet") + // $example off:generic_load_save_functions$ + // $example on:manual_load_options$ + val peopleDF = spark.read.format("json").load("examples/src/main/resources/people.json") + peopleDF.select("name", "age").write.format("parquet").save("namesAndAges.parquet") + // $example off:manual_load_options$ + // $example on:manual_load_options_csv$ + val peopleDFCsv = spark.read.format("csv") + .option("sep", ";") + .option("inferSchema", "true") + .option("header", "true") + .load("examples/src/main/resources/people.csv") + // $example off:manual_load_options_csv$ + // $example on:manual_save_options_orc$ + usersDF.write.format("orc") + .option("orc.bloom.filter.columns", "favorite_color") + .option("orc.dictionary.key.threshold", "1.0") + .option("orc.column.encoding.direct", "name") + .save("users_with_options.orc") + // $example off:manual_save_options_orc$ + // $example on:manual_save_options_parquet$ + usersDF.write.format("parquet") + .option("parquet.bloom.filter.enabled#favorite_color", "true") + .option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000") + .option("parquet.enable.dictionary", "true") + .option("parquet.page.write-checksum.enabled", "false") + .save("users_with_options.parquet") + // $example off:manual_save_options_parquet$ + + // $example on:direct_sql$ + val sqlDF = spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`") + // $example off:direct_sql$ + // $example on:write_sorting_and_bucketing$ + peopleDF.write.bucketBy(42, "name").sortBy("age").saveAsTable("people_bucketed") + // $example off:write_sorting_and_bucketing$ + // $example on:write_partitioning$ + usersDF.write.partitionBy("favorite_color").format("parquet").save("namesPartByColor.parquet") + // $example off:write_partitioning$ + // $example on:write_partition_and_bucket$ + usersDF + .write + .partitionBy("favorite_color") + .bucketBy(42, "name") + .saveAsTable("users_partitioned_bucketed") + // $example off:write_partition_and_bucket$ + + spark.sql("DROP TABLE IF EXISTS people_bucketed") + spark.sql("DROP TABLE IF EXISTS users_partitioned_bucketed") + } + + private def runBasicParquetExample(spark: SparkSession): Unit = { + // $example on:basic_parquet_example$ + // Encoders for most common types are automatically provided by importing spark.implicits._ + import spark.implicits._ + + val peopleDF = spark.read.json("examples/src/main/resources/people.json") + + // DataFrames can be saved as Parquet files, maintaining the schema information + peopleDF.write.parquet("people.parquet") + + // Read in the parquet file created above + // Parquet files are self-describing so the schema is preserved + // The result of loading a Parquet file is also a DataFrame + val parquetFileDF = spark.read.parquet("people.parquet") + + // Parquet files can also be used to create a temporary view and then used in SQL statements + parquetFileDF.createOrReplaceTempView("parquetFile") + val namesDF = spark.sql("SELECT name FROM parquetFile WHERE age BETWEEN 13 AND 19") + namesDF.map(attributes => "Name: " + attributes(0)).show() + // +------------+ + // | value| + // +------------+ + // |Name: Justin| + // +------------+ + // $example off:basic_parquet_example$ + } + + private def runParquetSchemaMergingExample(spark: SparkSession): Unit = { + // $example on:schema_merging$ + // This is used to implicitly convert an RDD to a DataFrame. + import spark.implicits._ + + // Create a simple DataFrame, store into a partition directory + val squaresDF = spark.sparkContext.makeRDD(1 to 5).map(i => (i, i * i)).toDF("value", "square") + squaresDF.write.parquet("data/test_table/key=1") + + // Create another DataFrame in a new partition directory, + // adding a new column and dropping an existing column + val cubesDF = spark.sparkContext.makeRDD(6 to 10).map(i => (i, i * i * i)).toDF("value", "cube") + cubesDF.write.parquet("data/test_table/key=2") + + // Read the partitioned table + val mergedDF = spark.read.option("mergeSchema", "true").parquet("data/test_table") + mergedDF.printSchema() + + // The final schema consists of all 3 columns in the Parquet files together + // with the partitioning column appeared in the partition directory paths + // root + // |-- value: int (nullable = true) + // |-- square: int (nullable = true) + // |-- cube: int (nullable = true) + // |-- key: int (nullable = true) + // $example off:schema_merging$ + } + + private def runJsonDatasetExample(spark: SparkSession): Unit = { + // $example on:json_dataset$ + // Primitive types (Int, String, etc) and Product types (case classes) encoders are + // supported by importing this when creating a Dataset. + import spark.implicits._ + + // A JSON dataset is pointed to by path. + // The path can be either a single text file or a directory storing text files + val path = "examples/src/main/resources/people.json" + val peopleDF = spark.read.json(path) + + // The inferred schema can be visualized using the printSchema() method + peopleDF.printSchema() + // root + // |-- age: long (nullable = true) + // |-- name: string (nullable = true) + + // Creates a temporary view using the DataFrame + peopleDF.createOrReplaceTempView("people") + + // SQL statements can be run by using the sql methods provided by spark + val teenagerNamesDF = spark.sql("SELECT name FROM people WHERE age BETWEEN 13 AND 19") + teenagerNamesDF.show() + // +------+ + // | name| + // +------+ + // |Justin| + // +------+ + + // Alternatively, a DataFrame can be created for a JSON dataset represented by + // a Dataset[String] storing one JSON object per string + val otherPeopleDataset = spark.createDataset( + """{"name":"Yin","address":{"city":"Columbus","state":"Ohio"}}""" :: Nil) + val otherPeople = spark.read.json(otherPeopleDataset) + otherPeople.show() + // +---------------+----+ + // | address|name| + // +---------------+----+ + // |[Columbus,Ohio]| Yin| + // +---------------+----+ + // $example off:json_dataset$ + } + + private def runCsvDatasetExample(spark: SparkSession): Unit = { + // $example on:csv_dataset$ + // A CSV dataset is pointed to by path. + // The path can be either a single CSV file or a directory of CSV files + val path = "examples/src/main/resources/people.csv" + + val df = spark.read.csv(path) + df.show() + // +------------------+ + // | _c0| + // +------------------+ + // | name;age;job| + // |Jorge;30;Developer| + // | Bob;32;Developer| + // +------------------+ + + // Read a csv with delimiter, the default delimiter is "," + val df2 = spark.read.option("delimiter", ";").csv(path) + df2.show() + // +-----+---+---------+ + // | _c0|_c1| _c2| + // +-----+---+---------+ + // | name|age| job| + // |Jorge| 30|Developer| + // | Bob| 32|Developer| + // +-----+---+---------+ + + // Read a csv with delimiter and a header + val df3 = spark.read.option("delimiter", ";").option("header", "true").csv(path) + df3.show() + // +-----+---+---------+ + // | name|age| job| + // +-----+---+---------+ + // |Jorge| 30|Developer| + // | Bob| 32|Developer| + // +-----+---+---------+ + + // You can also use options() to use multiple options + val df4 = spark.read.options(Map("delimiter" -> ";", "header" -> "true")).csv(path) + + // "output" is a folder which contains multiple csv files and a _SUCCESS file. + df3.write.csv("output") + + // Read all files in a folder, please make sure only CSV files should present in the folder. + val folderPath = "examples/src/main/resources"; + val df5 = spark.read.csv(folderPath); + df5.show(); + // Wrong schema because non-CSV files are read + // +-----------+ + // | _c0| + // +-----------+ + // |238val_238| + // | 86val_86| + // |311val_311| + // | 27val_27| + // |165val_165| + // +-----------+ + + // $example off:csv_dataset$ + } + + private def runTextDatasetExample(spark: SparkSession): Unit = { + // $example on:text_dataset$ + // A text dataset is pointed to by path. + // The path can be either a single text file or a directory of text files + val path = "examples/src/main/resources/people.txt" + + val df1 = spark.read.text(path) + df1.show() + // +-----------+ + // | value| + // +-----------+ + // |Michael, 29| + // | Andy, 30| + // | Justin, 19| + // +-----------+ + + // You can use 'lineSep' option to define the line separator. + // The line separator handles all `\r`, `\r\n` and `\n` by default. + val df2 = spark.read.option("lineSep", ",").text(path) + df2.show() + // +-----------+ + // | value| + // +-----------+ + // | Michael| + // | 29\nAndy| + // | 30\nJustin| + // | 19\n| + // +-----------+ + + // You can also use 'wholetext' option to read each input file as a single row. + val df3 = spark.read.option("wholetext", true).text(path) + df3.show() + // +--------------------+ + // | value| + // +--------------------+ + // |Michael, 29\nAndy...| + // +--------------------+ + + // "output" is a folder which contains multiple text files and a _SUCCESS file. + df1.write.text("output") + + // You can specify the compression format using the 'compression' option. + df1.write.option("compression", "gzip").text("output_compressed") + + // $example off:text_dataset$ + } + + private def runJdbcDatasetExample(spark: SparkSession): Unit = { + // $example on:jdbc_dataset$ + // Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods + // Loading data from a JDBC source + val jdbcDF = spark.read + .format("jdbc") + .option("url", "jdbc:postgresql://x.x.x.x:port/school") + .option("dbtable", "class") + .option("user", "sparkuser") + .option("password", "Enmo@123") + .load() + //.show() + + val connectionProperties = new Properties() + connectionProperties.put("user", "sparkuser") + connectionProperties.put("password", "Enmo@123") + val jdbcDF2 = spark.read + .option("customSchema","cla_id INT") + .jdbc("jdbc:postgresql://x.x.x.x:port/school", "class", connectionProperties).show() + // Specifying the custom data types of the read schema + + + connectionProperties.put("customSchema", "cla_id INT")//, cla_name STRING") + val jdbcDF3 = spark.read + .jdbc("jdbc:postgresql://x.x.x.x:port/school", "class", connectionProperties) + +// // Saving data to a JDBC source. Create table "customtable1", and write data +// jdbcDF.write +// .format("jdbc") +// .option("url", "jdbc:postgresql://x.x.x.x:port/school") +// .option("dbtable", "customtable1") +// .option("user", "sparkuser") +// .option("password", "Enmo@123") +// .save() +// +// +// jdbcDF2.write +// .jdbc("jdbc:postgresql://x.x.x.x:port/school", "customtable2", connectionProperties) + + // Specifying create table column data types on write + jdbcDF3.show() + +// jdbcDF3.write +// .option("createTableColumnTypes", "cla_id INT, cla_name VARCHAR(20)") +// .jdbc("jdbc:postgresql://x.x.x.x:port/school", "customtable3", connectionProperties) +// // $example off:jdbc_dataset$ + } +} \ No newline at end of file diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/multi/SimpleMultiDataSource.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/multi/SimpleMultiDataSource.scala new file mode 100644 index 00000000..a1c2a89e --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/multi/SimpleMultiDataSource.scala @@ -0,0 +1,102 @@ +package org.opengauss.spark.sources.datasourcev2.multi + +import java.util + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.read._ +import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ + +/* + * Default source should some kind of relation provider + */ +class DefaultSource extends TableProvider{ + + override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType = + getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema() + + override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table = + new SimpleBatchTable() + +} + + +/* + Defines Read Support and Initial Schema + */ + +class SimpleBatchTable extends Table with SupportsRead { + override def name(): String = this.getClass.toString + + override def schema(): StructType = StructType(Array(StructField("value", StringType))) + + override def capabilities(): util.Set[TableCapability] = Set(TableCapability.BATCH_READ).asJava + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder() +} + + +/* + Scan object with no mixins + */ +class SimpleScanBuilder extends ScanBuilder { + override def build(): Scan = new SimpleScan +} + +/* + Batch Reading Support + + The schema is repeated here as it can change after column pruning etc + */ + +class SimpleScan extends Scan with Batch{ + override def readSchema(): StructType = StructType(Array(StructField("value", StringType))) + + override def toBatch: Batch = this + + override def planInputPartitions(): Array[InputPartition] = { + Array(new SimplePartition(0,4), + new SimplePartition(5,9)) + } + override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory() +} + +// simple class to organise the partition +class SimplePartition(val start:Int, val end:Int) extends InputPartition + +// reader factory +class SimplePartitionReaderFactory extends PartitionReaderFactory { + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new + SimplePartitionReader(partition.asInstanceOf[SimplePartition]) +} + + +// parathion reader +class SimplePartitionReader(inputPartition: SimplePartition) extends PartitionReader[InternalRow] { + + val values = Array("1", "2", "3", "4", "5","6","7","8","9","10") + + var index = inputPartition.start + + def next = index <= inputPartition.end + + def get = { + val stringValue = values(index) + val stringUtf = UTF8String.fromString(stringValue) + val row = InternalRow(stringUtf) + index = index + 1 + row + } + + def close() = Unit + +} + + + + diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/simple/SimpleDataSource.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/simple/SimpleDataSource.scala new file mode 100644 index 00000000..0b8e0816 --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/simple/SimpleDataSource.scala @@ -0,0 +1,100 @@ +package org.opengauss.spark.sources.datasourcev2.simple + +import java.util + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.read._ +import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ + +/* + * Default source should some kind of relation provider + */ +class DefaultSource extends TableProvider{ + + override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType = + getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema() + + override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table = + new SimpleBatchTable() +} + + +/* + Defines Read Support and Initial Schema + */ + +class SimpleBatchTable extends Table with SupportsRead { + override def name(): String = this.getClass.toString + + override def schema(): StructType = StructType(Array(StructField("value", StringType))) + + override def capabilities(): util.Set[TableCapability] = Set(TableCapability.BATCH_READ).asJava + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder() +} + + + +/* + Scan object with no mixins + */ +class SimpleScanBuilder extends ScanBuilder { + override def build(): Scan = new SimpleScan +} + +/* + Batch Reading Support + + The schema is repeated here as it can change after column pruning etc + */ + +class SimpleScan extends Scan with Batch{ + override def readSchema(): StructType = StructType(Array(StructField("value", StringType))) + + override def toBatch: Batch = this + + override def planInputPartitions(): Array[InputPartition] = { + Array(new SimplePartition()) + } + override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory() +} + +// simple class to organise the partition +class SimplePartition extends InputPartition + +// reader factory +class SimplePartitionReaderFactory extends PartitionReaderFactory { + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new SimplePartitionReader +} + + +// parathion reader +class SimplePartitionReader extends PartitionReader[InternalRow] { + + val values = Array("1", "2", "3", "4", "5") + + var index = 0 + + def next = index < values.length + + def get = { + val stringValue = values(index) + val stringUtf = UTF8String.fromString(stringValue) + val row = InternalRow(stringUtf) + index = index + 1 + row + } + + def close() = Unit + +} + + + + diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.scala new file mode 100644 index 00000000..665ab712 --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.scala @@ -0,0 +1,37 @@ +package org.opengauss.spark.sources.datasourcev2.streamandbatch + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.streaming.OutputMode + +object DataSourceV2StreamAndBatchExample { + def main(args: Array[String]): Unit = { + val sparkSession = SparkSession.builder. + master("local[2]") + .appName("streaming example") + .getOrCreate() + + + val dataSource = "cn.ecnu.spark.sources.datasourcev2.streamandbatch.simple" + + val batchDf = sparkSession + .read + .format(dataSource) + .load() + + batchDf.show() + + val streamingDf = sparkSession. + readStream. + format(dataSource) + .load() + + val query = streamingDf.writeStream + .format("console") + .queryName("simple_source") + .outputMode(OutputMode.Append()) + + query.start().awaitTermination() + + } + +} diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamAndBatchDataSource.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamAndBatchDataSource.scala new file mode 100644 index 00000000..0fe5d52f --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamAndBatchDataSource.scala @@ -0,0 +1,130 @@ +package org.opengauss.spark.sources.datasourcev2.streamandbatch + +import java.util + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.read._ +import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset} +import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ + +/* + * Default source should some kind of relation provider + */ +class DefaultSource extends TableProvider{ + + override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType = + getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema() + + override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table = + new SimpleStreamingTable() +} + + +/* + Defines Read Support and Initial Schema + */ + +class SimpleStreamingTable extends Table with SupportsRead { + override def name(): String = this.getClass.toString + + override def schema(): StructType = StructType(Array(StructField("value", StringType))) + + override def capabilities(): util.Set[TableCapability] = Set(TableCapability.MICRO_BATCH_READ, + TableCapability.BATCH_READ).asJava + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder() +} + + +/* + Scan object with no mixins + */ +class SimpleScanBuilder extends ScanBuilder { + override def build(): Scan = new SimpleScan +} + +/* + Batch Reading Support + + The schema is repeated here as it can change after column pruning etc + */ + +class SimpleScan extends Scan{ + override def readSchema(): StructType = StructType(Array(StructField("value", StringType))) + + override def toMicroBatchStream(checkpointLocation: String): MicroBatchStream = new SimpleMicroBatchStream() + + override def toBatch: Batch = new SimpleBatch +} + +class SimpleBatch extends Batch{ + override def planInputPartitions(): Array[InputPartition] = Array(new SimplePartition) + + override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory +} + +class SimpleOffset(value:Int) extends Offset { + override def json(): String = s"""{"value":"$value"}""" +} + +class SimpleMicroBatchStream extends MicroBatchStream { + var latestOffsetValue = 0 + + override def latestOffset(): Offset = { + latestOffsetValue += 10 + new SimpleOffset(latestOffsetValue) + } + + override def planInputPartitions(offset: Offset, offset1: Offset): Array[InputPartition] = Array(new SimplePartition) + + override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory() + + override def initialOffset(): Offset = new SimpleOffset(latestOffsetValue) + + override def deserializeOffset(s: String): Offset = new SimpleOffset(latestOffsetValue) + + override def commit(offset: Offset): Unit = {} + + override def stop(): Unit = {} +} + + +// simple class to organise the partition +class SimplePartition extends InputPartition + +// reader factory +class SimplePartitionReaderFactory extends PartitionReaderFactory { + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new SimplePartitionReader +} + + +// parathion reader +class SimplePartitionReader extends PartitionReader[InternalRow] { + + val values = Array("1", "2", "3", "4", "5") + + var index = 0 + + def next = index < values.length + + def get = { + val stringValue = values(index) + val stringUtf = UTF8String.fromString(stringValue) + val row = InternalRow(stringUtf) + index = index + 1 + row + } + + def close() = Unit + +} + + + + diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.scala new file mode 100644 index 00000000..aebb6d79 --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.scala @@ -0,0 +1,29 @@ +package org.opengauss.spark.sources.datasourcev2.streaming + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.streaming.OutputMode + +object DataSourceV2StreamingExample { + def main(args: Array[String]): Unit = { + val sparkSession = SparkSession.builder. + master("local[2]") + .appName("streaming example") + .getOrCreate() + + val streamingDf = sparkSession. + readStream. + format("cn.ecnu.spark.sources.datasourcev2.streaming.simple") + .load() + + + + val query = streamingDf.writeStream + .format("console") + .queryName("simple_source") + .outputMode(OutputMode.Append()) + + query.start().awaitTermination() + + } + +} diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingDataSource.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingDataSource.scala new file mode 100644 index 00000000..89a37b8d --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingDataSource.scala @@ -0,0 +1,121 @@ +package org.opengauss.spark.sources.datasourcev2.streaming + +import java.util + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.read._ +import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset} +import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ + +/* + * Default source should some kind of relation provider + */ +class DefaultSource extends TableProvider{ + + override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType = + getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema() + + override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table = + new SimpleStreamingTable() +} + + +/* + Defines Read Support and Initial Schema + */ + +class SimpleStreamingTable extends Table with SupportsRead { + override def name(): String = this.getClass.toString + + override def schema(): StructType = StructType(Array(StructField("value", StringType))) + + override def capabilities(): util.Set[TableCapability] = Set(TableCapability.MICRO_BATCH_READ).asJava + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder() +} + + +/* + Scan object with no mixins + */ +class SimpleScanBuilder extends ScanBuilder { + override def build(): Scan = new SimpleScan +} + +/* + Batch Reading Support + + The schema is repeated here as it can change after column pruning etc + */ + +class SimpleScan extends Scan{ + override def readSchema(): StructType = StructType(Array(StructField("value", StringType))) + + override def toMicroBatchStream(checkpointLocation: String): MicroBatchStream = new SimpleMicroBatchStream() +} + +class SimpleOffset(value:Int) extends Offset { + override def json(): String = s"""{"value":"$value"}""" +} + +class SimpleMicroBatchStream extends MicroBatchStream { + var latestOffsetValue = 0 + + override def latestOffset(): Offset = { + latestOffsetValue += 10 + new SimpleOffset(latestOffsetValue) + } + + override def planInputPartitions(offset: Offset, offset1: Offset): Array[InputPartition] = Array(new SimplePartition) + + override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory() + + override def initialOffset(): Offset = new SimpleOffset(latestOffsetValue) + + override def deserializeOffset(s: String): Offset = new SimpleOffset(latestOffsetValue) + + override def commit(offset: Offset): Unit = {} + + override def stop(): Unit = {} +} + + +// simple class to organise the partition +class SimplePartition extends InputPartition + +// reader factory +class SimplePartitionReaderFactory extends PartitionReaderFactory { + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new SimplePartitionReader +} + + +// parathion reader +class SimplePartitionReader extends PartitionReader[InternalRow] { + + val values = Array("1", "2", "3", "4", "5") + + var index = 0 + + def next = index < values.length + + def get = { + val stringValue = values(index) + val stringUtf = UTF8String.fromString(stringValue) + val row = InternalRow(stringUtf) + index = index + 1 + row + } + + def close() = Unit + +} + + + + diff --git a/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/opengauss/OpenGaussDataSource.scala b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/opengauss/OpenGaussDataSource.scala new file mode 100644 index 00000000..89625577 --- /dev/null +++ b/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/opengauss/OpenGaussDataSource.scala @@ -0,0 +1,152 @@ +package org.opengauss.spark.sources.opengauss + +import java.sql.DriverManager +import java.util + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.catalog._ +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.connector.read._ +import org.apache.spark.sql.connector.write._ +import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, StructField, StructType} +import org.apache.spark.sql.util.CaseInsensitiveStringMap +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ + +class DefaultSource extends TableProvider { + override def inferSchema(options: CaseInsensitiveStringMap): StructType = OpenGaussTable.schema + + override def getTable( + schema: StructType, + partitioning: Array[Transform], + properties: util.Map[String, String] + ): Table = new OpenGaussTable(properties.get("tableName")) // TODO: Error handling +} + +class OpenGaussTable(val name: String) extends SupportsRead with SupportsWrite { + override def schema(): StructType = OpenGaussTable.schema + + override def capabilities(): util.Set[TableCapability] = Set( + TableCapability.BATCH_READ, + TableCapability.BATCH_WRITE + ).asJava + + override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new OpenGaussScanBuilder(options) + + override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = new OpenGaussWriteBuilder(info.options) +} + +object OpenGaussTable { + /*Table products*/ + /*Database school, table course*/ + val schema: StructType = new StructType().add("cor_id", IntegerType).add("cor_name", StringType).add("cor_type", StringType).add("credit", DoubleType) +} + +case class ConnectionProperties(url: String, user: String, password: String, tableName: String, partitionColumn: String, partitionSize: Int) + + + +/** Read */ + +class OpenGaussScanBuilder(options: CaseInsensitiveStringMap) extends ScanBuilder { + override def build(): Scan = new OpenGaussScan(ConnectionProperties( + options.get("url"), options.get("user"), options.get("password"), options.get("tableName"), options.get("partitionColumn"), options.get("partitionSize").toInt + )) +} + +class OpenGaussPartition extends InputPartition + +class OpenGaussScan(connectionProperties: ConnectionProperties) extends Scan with Batch { + override def readSchema(): StructType = OpenGaussTable.schema + + override def toBatch: Batch = this + + override def planInputPartitions(): Array[InputPartition] = Array(new OpenGaussPartition) + + override def createReaderFactory(): PartitionReaderFactory = new OpenGaussPartitionReaderFactory(connectionProperties) +} + +class OpenGaussPartitionReaderFactory(connectionProperties: ConnectionProperties) + extends PartitionReaderFactory { + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new OpenGaussPartitionReader(connectionProperties) +} + + + +class OpenGaussPartitionReader(connectionProperties: ConnectionProperties) extends PartitionReader[InternalRow] { + private val connection = DriverManager.getConnection( + connectionProperties.url, connectionProperties.user, connectionProperties.password + ) + private val statement = connection.createStatement() + private val resultSet = statement.executeQuery(s"select * from ${connectionProperties.tableName}") + + override def next(): Boolean = resultSet.next() + + override def get(): InternalRow = InternalRow( + resultSet.getInt(1), + UTF8String.fromString(resultSet.getString(2)), + UTF8String.fromString(resultSet.getString(3)), + resultSet.getDouble(4)) + + override def close(): Unit = connection.close() + +} + + + +/** Write */ + +class OpenGaussWriteBuilder(options: CaseInsensitiveStringMap) extends WriteBuilder { + override def buildForBatch(): BatchWrite = new OpenGaussBatchWrite(ConnectionProperties( + options.get("url"), options.get("user"), options.get("password"), options.get("tableName"), options.get("partitionColumn"), options.get("partitionSize").toInt + )) +} + +class OpenGaussBatchWrite(connectionProperties: ConnectionProperties) extends BatchWrite { + override def createBatchWriterFactory(physicalWriteInfo: PhysicalWriteInfo): DataWriterFactory = + new OpenGaussDataWriterFactory(connectionProperties) + + override def commit(writerCommitMessages: Array[WriterCommitMessage]): Unit = {} + + override def abort(writerCommitMessages: Array[WriterCommitMessage]): Unit = {} +} + +class OpenGaussDataWriterFactory(connectionProperties: ConnectionProperties) extends DataWriterFactory { + override def createWriter(partitionId: Int, taskId:Long): DataWriter[InternalRow] = + new OpenGaussWriter(connectionProperties) +} + +object WriteSucceeded extends WriterCommitMessage + +class OpenGaussWriter(connectionProperties: ConnectionProperties) extends DataWriter[InternalRow] { + + val connection = DriverManager.getConnection( + connectionProperties.url, + connectionProperties.user, + connectionProperties.password + ) + + // TODO：待修改 + val statement = "insert into ${connectionProperties.tableName} (cor_name, cor_type, credit) values (?,?,?)" + val preparedStatement = connection.prepareStatement(statement) + + override def write(record: InternalRow): Unit = { + val cor_name = record.getString(0) + val cor_type = record.getString(1) + val credit = record.getDouble(2) + + preparedStatement.setString(0, cor_name) + preparedStatement.setString(1, cor_type) + preparedStatement.setDouble(2, credit) + preparedStatement.executeUpdate() + } + + override def commit(): WriterCommitMessage = WriteSucceeded + + override def abort(): Unit = {} + + override def close(): Unit = connection.close() +} + + diff --git a/SparkOpOpenGauss/src/test/scala/org/opengauss/spark/OpenGaussExample.scala b/SparkOpOpenGauss/src/test/scala/org/opengauss/spark/OpenGaussExample.scala new file mode 100644 index 00000000..53ad067b --- /dev/null +++ b/SparkOpOpenGauss/src/test/scala/org/opengauss/spark/OpenGaussExample.scala @@ -0,0 +1,97 @@ +package org.opengauss.spark + +import org.apache.spark.sql.{SaveMode, SparkSession} +import org.scalatest.FlatSpec +import java.sql.DriverManager +import java.util.Properties + +import org.scalatest.Matchers.convertToAnyShouldWrapper + +class OpenGaussExample extends FlatSpec { + + val testTableName = "course" + + "Simple data source" should "read" in{ + val sparkSession = SparkSession.builder + .master("local[2]") + .appName("example") + .getOrCreate() + + val simpleDf = sparkSession.read + .format("cn.ecnu.spark.sources.datasourcev2.simple") + .load() + + simpleDf.show() + println( + "number of partitions in simple source is " + simpleDf.rdd.getNumPartitions) + } + + + "openGauss data source" should "read table" in { + val spark = SparkSession + .builder() + .master("local[*]") + .appName("OpenGaussReaderJob") + .getOrCreate() + + val simpleRead = spark + .read + .format("org.opengauss.spark.sources.opengauss") + .option("url", "jdbc:postgresql://x.x.x.x:port/school") + .option("user", "sparkuser") + .option("password", "Enmo@123") + .option("tableName", testTableName) + .option("partitionSize", 10) +// .option("partitionColumn", "name") + .load() + .show() + + spark.stop() + } + + "openGauss data source" should "write table" in { + val spark = SparkSession + .builder() + .master("local[*]") + .appName("OpenGaussWriterJob") + .getOrCreate() + + import spark.implicits._ + + val df = (60 to 70).map(_.toLong).toDF("product_no") + + df + .write + .format("org.opengauss.spark.sources.opengauss") + .option("url", "jdbc:postgresql://x.x.x.x:port/postgres") + .option("user", "sparkuser") + .option("password", "Enmo@123") + .option("tableName", testTableName) + .option("partitionSize", 10) + .option("partitionColumn", "product_no") + .mode(SaveMode.Append) + .save() + + spark.stop() + } + + + +// def connection(c: PostgreSQLContainer) = { +// Class.forName(c.driverClassName) +// val properties = new Properties() +// properties.put("user", c.username) +// properties.put("password", c.password) +// DriverManager.getConnection(c.jdbcUrl, properties) +// } + + object Queries { + lazy val createTableQuery = s"CREATE TABLE $testTableName (user_id BIGINT PRIMARY KEY);" + + lazy val testValues: String = (1 to 50).map(i => s"($i)").mkString(", ") + + lazy val insertDataQuery = s"INSERT INTO $testTableName VALUES $testValues;" + } + +} + diff --git a/SparkOpOpenGauss/target/classes/adult.csv b/SparkOpOpenGauss/target/classes/adult.csv new file mode 100644 index 00000000..2aae7c3d --- /dev/null +++ b/SparkOpOpenGauss/target/classes/adult.csv @@ -0,0 +1,102 @@ +age,workclass,fnlwgt,education,education_num,martial_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,salary +39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White, Male, 2174, 0, 40, United-States, <=50K +50, Self-emp-not-inc, 83311, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 13, United-States, <=50K +38, Private, 215646, HS-grad, 9, Divorced, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K +53, Private, 234721, 11th, 7, Married-civ-spouse, Handlers-cleaners, Husband, Black, Male, 0, 0, 40, United-States, <=50K +28, Private, 338409, Bachelors, 13, Married-civ-spouse, Prof-specialty, Wife, Black, Female, 0, 0, 40, Cuba, <=50K +37, Private, 284582, Masters, 14, Married-civ-spouse, Exec-managerial, Wife, White, Female, 0, 0, 40, United-States, <=50K +49, Private, 160187, 9th, 5, Married-spouse-absent, Other-service, Not-in-family, Black, Female, 0, 0, 16, Jamaica, <=50K +52, Self-emp-not-inc, 209642, HS-grad, 9, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 45, United-States, >50K +31, Private, 45781, Masters, 14, Never-married, Prof-specialty, Not-in-family, White, Female, 14084, 0, 50, United-States, >50K +42, Private, 159449, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 5178, 0, 40, United-States, >50K +37, Private, 280464, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, Black, Male, 0, 0, 80, United-States, >50K +30, State-gov, 141297, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Asian-Pac-Islander, Male, 0, 0, 40, India, >50K +23, Private, 122272, Bachelors, 13, Never-married, Adm-clerical, Own-child, White, Female, 0, 0, 30, United-States, <=50K +32, Private, 205019, Assoc-acdm, 12, Never-married, Sales, Not-in-family, Black, Male, 0, 0, 50, United-States, <=50K +40, Private, 121772, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male, 0, 0, 40, ?, >50K +34, Private, 245487, 7th-8th, 4, Married-civ-spouse, Transport-moving, Husband, Amer-Indian-Eskimo, Male, 0, 0, 45, Mexico, <=50K +25, Self-emp-not-inc, 176756, HS-grad, 9, Never-married, Farming-fishing, Own-child, White, Male, 0, 0, 35, United-States, <=50K +32, Private, 186824, HS-grad, 9, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, United-States, <=50K +38, Private, 28887, 11th, 7, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 50, United-States, <=50K +43, Self-emp-not-inc, 292175, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 45, United-States, >50K +40, Private, 193524, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 60, United-States, >50K +54, Private, 302146, HS-grad, 9, Separated, Other-service, Unmarried, Black, Female, 0, 0, 20, United-States, <=50K +35, Federal-gov, 76845, 9th, 5, Married-civ-spouse, Farming-fishing, Husband, Black, Male, 0, 0, 40, United-States, <=50K +43, Private, 117037, 11th, 7, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 2042, 40, United-States, <=50K +59, Private, 109015, HS-grad, 9, Divorced, Tech-support, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +56, Local-gov, 216851, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K +19, Private, 168294, HS-grad, 9, Never-married, Craft-repair, Own-child, White, Male, 0, 0, 40, United-States, <=50K +54, ?, 180211, Some-college, 10, Married-civ-spouse, ?, Husband, Asian-Pac-Islander, Male, 0, 0, 60, South, >50K +39, Private, 367260, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 80, United-States, <=50K +49, Private, 193366, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K +23, Local-gov, 190709, Assoc-acdm, 12, Never-married, Protective-serv, Not-in-family, White, Male, 0, 0, 52, United-States, <=50K +20, Private, 266015, Some-college, 10, Never-married, Sales, Own-child, Black, Male, 0, 0, 44, United-States, <=50K +45, Private, 386940, Bachelors, 13, Divorced, Exec-managerial, Own-child, White, Male, 0, 1408, 40, United-States, <=50K +30, Federal-gov, 59951, Some-college, 10, Married-civ-spouse, Adm-clerical, Own-child, White, Male, 0, 0, 40, United-States, <=50K +22, State-gov, 311512, Some-college, 10, Married-civ-spouse, Other-service, Husband, Black, Male, 0, 0, 15, United-States, <=50K +48, Private, 242406, 11th, 7, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, Puerto-Rico, <=50K +21, Private, 197200, Some-college, 10, Never-married, Machine-op-inspct, Own-child, White, Male, 0, 0, 40, United-States, <=50K +19, Private, 544091, HS-grad, 9, Married-AF-spouse, Adm-clerical, Wife, White, Female, 0, 0, 25, United-States, <=50K +31, Private, 84154, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 38, ?, >50K +48, Self-emp-not-inc, 265477, Assoc-acdm, 12, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K +31, Private, 507875, 9th, 5, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 43, United-States, <=50K +53, Self-emp-not-inc, 88506, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K +24, Private, 172987, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 50, United-States, <=50K +49, Private, 94638, HS-grad, 9, Separated, Adm-clerical, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +25, Private, 289980, HS-grad, 9, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 35, United-States, <=50K +57, Federal-gov, 337895, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Black, Male, 0, 0, 40, United-States, >50K +53, Private, 144361, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 38, United-States, <=50K +44, Private, 128354, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +41, State-gov, 101603, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K +29, Private, 271466, Assoc-voc, 11, Never-married, Prof-specialty, Not-in-family, White, Male, 0, 0, 43, United-States, <=50K +25, Private, 32275, Some-college, 10, Married-civ-spouse, Exec-managerial, Wife, Other, Female, 0, 0, 40, United-States, <=50K +18, Private, 226956, HS-grad, 9, Never-married, Other-service, Own-child, White, Female, 0, 0, 30, ?, <=50K +47, Private, 51835, Prof-school, 15, Married-civ-spouse, Prof-specialty, Wife, White, Female, 0, 1902, 60, Honduras, >50K +50, Federal-gov, 251585, Bachelors, 13, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 55, United-States, >50K +47, Self-emp-inc, 109832, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 60, United-States, <=50K +43, Private, 237993, Some-college, 10, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K +46, Private, 216666, 5th-6th, 3, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 40, Mexico, <=50K +35, Private, 56352, Assoc-voc, 11, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Puerto-Rico, <=50K +41, Private, 147372, HS-grad, 9, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 48, United-States, <=50K +30, Private, 188146, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 5013, 0, 40, United-States, <=50K +30, Private, 59496, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 2407, 0, 40, United-States, <=50K +32, ?, 293936, 7th-8th, 4, Married-spouse-absent, ?, Not-in-family, White, Male, 0, 0, 40, ?, <=50K +48, Private, 149640, HS-grad, 9, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 0, 40, United-States, <=50K +42, Private, 116632, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 45, United-States, >50K +29, Private, 105598, Some-college, 10, Divorced, Tech-support, Not-in-family, White, Male, 0, 0, 58, United-States, <=50K +36, Private, 155537, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K +28, Private, 183175, Some-college, 10, Divorced, Adm-clerical, Not-in-family, White, Female, 0, 0, 40, United-States, <=50K +53, Private, 169846, HS-grad, 9, Married-civ-spouse, Adm-clerical, Wife, White, Female, 0, 0, 40, United-States, >50K +49, Self-emp-inc, 191681, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 50, United-States, >50K +25, ?, 200681, Some-college, 10, Never-married, ?, Own-child, White, Male, 0, 0, 40, United-States, <=50K +19, Private, 101509, Some-college, 10, Never-married, Prof-specialty, Own-child, White, Male, 0, 0, 32, United-States, <=50K +31, Private, 309974, Bachelors, 13, Separated, Sales, Own-child, Black, Female, 0, 0, 40, United-States, <=50K +29, Self-emp-not-inc, 162298, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 70, United-States, >50K +23, Private, 211678, Some-college, 10, Never-married, Machine-op-inspct, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K +79, Private, 124744, Some-college, 10, Married-civ-spouse, Prof-specialty, Other-relative, White, Male, 0, 0, 20, United-States, <=50K +27, Private, 213921, HS-grad, 9, Never-married, Other-service, Own-child, White, Male, 0, 0, 40, Mexico, <=50K +40, Private, 32214, Assoc-acdm, 12, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 40, United-States, <=50K +67, ?, 212759, 10th, 6, Married-civ-spouse, ?, Husband, White, Male, 0, 0, 2, United-States, <=50K +18, Private, 309634, 11th, 7, Never-married, Other-service, Own-child, White, Female, 0, 0, 22, United-States, <=50K +31, Local-gov, 125927, 7th-8th, 4, Married-civ-spouse, Farming-fishing, Husband, White, Male, 0, 0, 40, United-States, <=50K +18, Private, 446839, HS-grad, 9, Never-married, Sales, Not-in-family, White, Male, 0, 0, 30, United-States, <=50K +52, Private, 276515, Bachelors, 13, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Cuba, <=50K +46, Private, 51618, HS-grad, 9, Married-civ-spouse, Other-service, Wife, White, Female, 0, 0, 40, United-States, <=50K +59, Private, 159937, HS-grad, 9, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, <=50K +44, Private, 343591, HS-grad, 9, Divorced, Craft-repair, Not-in-family, White, Female, 14344, 0, 40, United-States, >50K +53, Private, 346253, HS-grad, 9, Divorced, Sales, Own-child, White, Female, 0, 0, 35, United-States, <=50K +49, Local-gov, 268234, HS-grad, 9, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K +33, Private, 202051, Masters, 14, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 50, United-States, <=50K +30, Private, 54334, 9th, 5, Never-married, Sales, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K +43, Federal-gov, 410867, Doctorate, 16, Never-married, Prof-specialty, Not-in-family, White, Female, 0, 0, 50, United-States, >50K +57, Private, 249977, Assoc-voc, 11, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K +37, Private, 286730, Some-college, 10, Divorced, Craft-repair, Unmarried, White, Female, 0, 0, 40, United-States, <=50K +28, Private, 212563, Some-college, 10, Divorced, Machine-op-inspct, Unmarried, Black, Female, 0, 0, 25, United-States, <=50K +30, Private, 117747, HS-grad, 9, Married-civ-spouse, Sales, Wife, Asian-Pac-Islander, Female, 0, 1573, 35, ?, <=50K +34, Local-gov, 226296, Bachelors, 13, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K +29, Local-gov, 115585, Some-college, 10, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 50, United-States, <=50K +48, Self-emp-not-inc, 191277, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 1902, 60, United-States, >50K +37, Private, 202683, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, >50K +48, Private, 171095, Assoc-acdm, 12, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, England, <=50K +32, Federal-gov, 249409, HS-grad, 9, Never-married, Other-service, Own-child, Black, Male, 0, 0, 40, United-States, <=50K +76, Private, 124191, Masters, 14, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 40, United-States, >50K diff --git a/SparkOpOpenGauss/target/classes/customers.csv b/SparkOpOpenGauss/target/classes/customers.csv new file mode 100644 index 00000000..0b4f8528 --- /dev/null +++ b/SparkOpOpenGauss/target/classes/customers.csv @@ -0,0 +1,5 @@ +customerId,customerName +1,John +2,Clerk +3,Micheal +4,Sample diff --git a/SparkOpOpenGauss/target/classes/metrics.properties b/SparkOpOpenGauss/target/classes/metrics.properties new file mode 100644 index 00000000..a882870e --- /dev/null +++ b/SparkOpOpenGauss/target/classes/metrics.properties @@ -0,0 +1 @@ +*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink \ No newline at end of file diff --git a/SparkOpOpenGauss/target/classes/multicharacterseperator.csv b/SparkOpOpenGauss/target/classes/multicharacterseperator.csv new file mode 100644 index 00000000..9db32e5b --- /dev/null +++ b/SparkOpOpenGauss/target/classes/multicharacterseperator.csv @@ -0,0 +1,3 @@ +a||b||c||d +1||2||3||4 +5||6||7||8 \ No newline at end of file diff --git a/SparkOpOpenGauss/target/classes/nested/a.csv b/SparkOpOpenGauss/target/classes/nested/a.csv new file mode 100644 index 00000000..9db32e5b --- /dev/null +++ b/SparkOpOpenGauss/target/classes/nested/a.csv @@ -0,0 +1,3 @@ +a||b||c||d +1||2||3||4 +5||6||7||8 \ No newline at end of file diff --git a/SparkOpOpenGauss/target/classes/nested/folder1/b.csv b/SparkOpOpenGauss/target/classes/nested/folder1/b.csv new file mode 100644 index 00000000..9db32e5b --- /dev/null +++ b/SparkOpOpenGauss/target/classes/nested/folder1/b.csv @@ -0,0 +1,3 @@ +a||b||c||d +1||2||3||4 +5||6||7||8 \ No newline at end of file diff --git a/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator16$1.class b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator16$1.class new file mode 100644 index 00000000..f77a630d Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator16$1.class differ diff --git a/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator6$1.class b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator6$1.class new file mode 100644 index 00000000..d872c15e Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator6$1.class differ diff --git a/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$.class b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$.class new file mode 100644 index 00000000..d42d130c Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person$.class b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person$.class new file mode 100644 index 00000000..451c5609 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person.class b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person.class new file mode 100644 index 00000000..42032d9a Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person.class differ diff --git a/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample.class b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample.class new file mode 100644 index 00000000..ef6bac06 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/DefaultSource.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/DefaultSource.class new file mode 100644 index 00000000..16ad16b7 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/DefaultSource.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleBatchTable.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleBatchTable.class new file mode 100644 index 00000000..776b3109 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleBatchTable.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartition.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartition.class new file mode 100644 index 00000000..b301508c Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartition.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReader.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReader.class new file mode 100644 index 00000000..d36c13c3 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReader.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReaderFactory.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReaderFactory.class new file mode 100644 index 00000000..a4b5022d Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReaderFactory.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScan.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScan.class new file mode 100644 index 00000000..f52c07ab Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScan.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScanBuilder.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScanBuilder.class new file mode 100644 index 00000000..41d3b315 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScanBuilder.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/DefaultSource.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/DefaultSource.class new file mode 100644 index 00000000..966a613c Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/DefaultSource.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleBatchTable.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleBatchTable.class new file mode 100644 index 00000000..88bb21f6 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleBatchTable.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartition.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartition.class new file mode 100644 index 00000000..3862f3f9 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartition.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReader.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReader.class new file mode 100644 index 00000000..d1c1481f Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReader.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReaderFactory.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReaderFactory.class new file mode 100644 index 00000000..3d5d337f Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReaderFactory.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScan.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScan.class new file mode 100644 index 00000000..e6ad939b Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScan.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScanBuilder.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScanBuilder.class new file mode 100644 index 00000000..e91ef9b8 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScanBuilder.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample$.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample$.class new file mode 100644 index 00000000..dccb9ca1 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.class new file mode 100644 index 00000000..b6bc0966 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DefaultSource.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DefaultSource.class new file mode 100644 index 00000000..e8e5e9bf Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DefaultSource.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleBatch.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleBatch.class new file mode 100644 index 00000000..b4fdca4b Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleBatch.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleMicroBatchStream.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleMicroBatchStream.class new file mode 100644 index 00000000..e1e17d7d Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleMicroBatchStream.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleOffset.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleOffset.class new file mode 100644 index 00000000..20c9ae68 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleOffset.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartition.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartition.class new file mode 100644 index 00000000..d33ddf32 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartition.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReader.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReader.class new file mode 100644 index 00000000..44078c0c Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReader.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReaderFactory.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReaderFactory.class new file mode 100644 index 00000000..0ae0a7c6 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReaderFactory.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScan.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScan.class new file mode 100644 index 00000000..a0c7f5c3 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScan.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScanBuilder.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScanBuilder.class new file mode 100644 index 00000000..a4fde431 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScanBuilder.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamingTable.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamingTable.class new file mode 100644 index 00000000..348c8c76 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamingTable.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample$.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample$.class new file mode 100644 index 00000000..24661f2d Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.class new file mode 100644 index 00000000..b7f93ada Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DefaultSource.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DefaultSource.class new file mode 100644 index 00000000..1d84d959 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DefaultSource.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleMicroBatchStream.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleMicroBatchStream.class new file mode 100644 index 00000000..e6ad3411 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleMicroBatchStream.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleOffset.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleOffset.class new file mode 100644 index 00000000..30536a2a Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleOffset.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartition.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartition.class new file mode 100644 index 00000000..cc94484e Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartition.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReader.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReader.class new file mode 100644 index 00000000..e5ad1364 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReader.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReaderFactory.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReaderFactory.class new file mode 100644 index 00000000..fd9b46c6 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReaderFactory.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScan.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScan.class new file mode 100644 index 00000000..16565704 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScan.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScanBuilder.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScanBuilder.class new file mode 100644 index 00000000..9940bfc7 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScanBuilder.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingTable.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingTable.class new file mode 100644 index 00000000..d0851a90 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingTable.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties$.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties$.class new file mode 100644 index 00000000..082890ea Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties.class new file mode 100644 index 00000000..a1e5d720 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/DefaultSource.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/DefaultSource.class new file mode 100644 index 00000000..21559b2c Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/DefaultSource.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussBatchWrite.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussBatchWrite.class new file mode 100644 index 00000000..e4c7d071 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussBatchWrite.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussDataWriterFactory.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussDataWriterFactory.class new file mode 100644 index 00000000..bbb9fa09 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussDataWriterFactory.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartition.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartition.class new file mode 100644 index 00000000..b5692a05 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartition.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReader.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReader.class new file mode 100644 index 00000000..c371d270 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReader.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReaderFactory.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReaderFactory.class new file mode 100644 index 00000000..d740f1ea Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReaderFactory.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScan.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScan.class new file mode 100644 index 00000000..3d9a4cb4 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScan.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScanBuilder.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScanBuilder.class new file mode 100644 index 00000000..712efa27 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScanBuilder.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable$.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable$.class new file mode 100644 index 00000000..0d45edab Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable.class new file mode 100644 index 00000000..905a815e Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriteBuilder.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriteBuilder.class new file mode 100644 index 00000000..42858e7f Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriteBuilder.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriter.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriter.class new file mode 100644 index 00000000..8c237066 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriter.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded$.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded$.class new file mode 100644 index 00000000..d54d7813 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded$.class differ diff --git a/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded.class b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded.class new file mode 100644 index 00000000..3d770329 Binary files /dev/null and b/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded.class differ diff --git a/SparkOpOpenGauss/target/classes/sales.csv b/SparkOpOpenGauss/target/classes/sales.csv new file mode 100644 index 00000000..b3145638 --- /dev/null +++ b/SparkOpOpenGauss/target/classes/sales.csv @@ -0,0 +1,15 @@ +transactionId,customerId,itemId,amountPaid +111,1,1,100.0 +112,2,2,505.0 +113,3,3,510.0 +114,4,4,600.0 +115,1,2,500.0 +116,1,2,500.0 +117,1,2,500.0 +118,1,2,500.0 +119,2,3,500.0 +120,1,2,500.0 +121,1,4,500.0 +122,1,2,500.0 +123,1,4,500.0 +124,1,2,500.0 \ No newline at end of file diff --git a/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample$Queries$.class b/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample$Queries$.class new file mode 100644 index 00000000..3ca4fed0 Binary files /dev/null and b/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample$Queries$.class differ diff --git a/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample.class b/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample.class new file mode 100644 index 00000000..741faac5 Binary files /dev/null and b/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample.class differ