forked from opengaussexamples/examples
parent
8e88d94be1
commit
0298cc3f4a
|
|
@ -0,0 +1,9 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
||||
<exclude-output />
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/examples.iml" filepath="$PROJECT_DIR$/.idea/examples.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="66fc4f6b-64dc-4070-be20-4baa614dcd2b" name="Default Changelist" comment="">
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/.gitignore" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/SparkOpOGForTest.iml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/SparkOpOpenGauss.iml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/codeStyles/Project.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/codeStyles/codeStyleConfig.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/compiler.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/information_schema.FNRwLQ.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/pg_catalog.0S1ZNQ.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/postgres.edMnLQ/schema/sparkuser.KCgh4Q.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/information_schema.FNRwLQ.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/dataSources/f5515d82-d7ba-4fc5-9425-f44775711cf3/storage_v2/_src_/database/school.dFvhyQ/schema/pg_catalog.0S1ZNQ.meta" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/google-java-format.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/jarRepositories.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/libraries/opengauss_jdbc_2_0_0.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/misc.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/uiDesigner.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/.idea/vcs.xml" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/README.md" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/libs/opengauss-jdbc-2.0.0.jar" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/pom.xml" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/pom.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/adult.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/customers.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/metrics.properties" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/multicharacterseperator.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/nested/a.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/nested/folder1/b.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/resources/sales.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/SQLDataSourceExample.scala" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/SQLDataSourceExample.scala" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/multi/SimpleMultiDataSource.scala" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/multi/SimpleMultiDataSource.scala" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.scala" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.scala" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.scala" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/src/main/scala/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.scala" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/src/test/scala/org/opengauss/spark/OpenGaussExample.scala" beforeDir="false" afterPath="$PROJECT_DIR$/SparkOpOpenGauss/src/test/scala/org/opengauss/spark/OpenGaussExample.scala" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/adult.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/customers.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/metrics.properties" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/multicharacterseperator.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/nested/a.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/nested/folder1/b.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator16$1.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$$typecreator6$1.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample$Person.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/apache/spark/examples/sql/SQLDataSourceExample.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/DefaultSource.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleBatchTable.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartition.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReader.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimplePartitionReaderFactory.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScan.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/multi/SimpleScanBuilder.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/DefaultSource.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleBatchTable.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartition.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReader.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimplePartitionReaderFactory.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScan.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/simple/SimpleScanBuilder.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DataSourceV2StreamAndBatchExample.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/DefaultSource.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleBatch.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleMicroBatchStream.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleOffset.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartition.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReader.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimplePartitionReaderFactory.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScan.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleScanBuilder.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streamandbatch/SimpleStreamingTable.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DataSourceV2StreamingExample.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/DefaultSource.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleMicroBatchStream.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleOffset.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartition.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReader.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimplePartitionReaderFactory.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScan.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleScanBuilder.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/datasourcev2/streaming/SimpleStreamingTable.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/ConnectionProperties.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/DefaultSource.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussBatchWrite.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussDataWriterFactory.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartition.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReader.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussPartitionReaderFactory.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScan.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussScanBuilder.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussTable.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriteBuilder.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/OpenGaussWriter.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/org/opengauss/spark/sources/opengauss/WriteSucceeded.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/classes/sales.csv" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample$Queries$.class" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/SparkOpOpenGauss/target/test-classes/org/opengauss/spark/OpenGaussExample.class" beforeDir="false" />
|
||||
</list>
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
<option name="HIGHLIGHT_CONFLICTS" value="true" />
|
||||
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
|
||||
<option name="LAST_RESOLUTION" value="IGNORE" />
|
||||
</component>
|
||||
<component name="CodeStyleSettingsInfer">
|
||||
<option name="done" value="true" />
|
||||
</component>
|
||||
<component name="Git.Settings">
|
||||
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
|
||||
</component>
|
||||
<component name="MavenImportPreferences">
|
||||
<option name="generalSettings">
|
||||
<MavenGeneralSettings>
|
||||
<option name="mavenHome" value="C:\Users\Cerdore\IntelliJ IDEA 2020.2.3\plugins\maven\lib\maven3" />
|
||||
</MavenGeneralSettings>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectCodeStyleSettingsMigration">
|
||||
<option name="version" value="1" />
|
||||
</component>
|
||||
<component name="ProjectId" id="1yijxYGvIfqqhpfuIDCkg0dPCLD" />
|
||||
<component name="ProjectViewState">
|
||||
<option name="hideEmptyMiddlePackages" value="true" />
|
||||
<option name="showLibraryContents" value="true" />
|
||||
</component>
|
||||
<component name="PropertiesComponent">
|
||||
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
|
||||
<property name="WebServerToolWindowFactoryState" value="false" />
|
||||
<property name="aspect.path.notification.shown" value="true" />
|
||||
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
|
||||
</component>
|
||||
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
|
||||
<component name="TaskManager">
|
||||
<task active="true" id="Default" summary="Default task">
|
||||
<changelist id="66fc4f6b-64dc-4070-be20-4baa614dcd2b" name="Default Changelist" comment="" />
|
||||
<created>1632741549572</created>
|
||||
<option name="number" value="Default" />
|
||||
<option name="presentableId" value="Default" />
|
||||
<updated>1632741549572</updated>
|
||||
<workItem from="1632741550670" duration="2986000" />
|
||||
</task>
|
||||
<servers />
|
||||
</component>
|
||||
<component name="TypeScriptGeneratedFilesManager">
|
||||
<option name="version" value="3" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Datasource local storage ignored files
|
||||
/../../../../:\gitRepo\SparkOpOG\.idea/dataSources/
|
||||
/dataSources.local.xml
|
||||
# Zeppelin ignored files
|
||||
/ZeppelinRemoteNotebooks/
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CompilerConfiguration">
|
||||
<annotationProcessing>
|
||||
<profile name="Maven default annotation processors profile" enabled="true">
|
||||
<sourceOutputDir name="target/generated-sources/annotations" />
|
||||
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
|
||||
<outputRelativeToContentRoot value="true" />
|
||||
<module name="SparkOpOpenGauss" />
|
||||
<module name="SparkOpOG" />
|
||||
<module name="SparkOpOGForTest" />
|
||||
</profile>
|
||||
</annotationProcessing>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="opengauss@10.11.6.27" uuid="f5515d82-d7ba-4fc5-9425-f44775711cf3">
|
||||
<driver-ref>postgresql</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
|
||||
<jdbc-url>jdbc:postgresql://10.11.6.27:15432/postgres</jdbc-url>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1 +0,0 @@
|
|||
#n:postgres
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
#n:information_schema
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
#n:pg_catalog
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
#n:sparkuser
|
||||
!<md> [11790, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -1 +0,0 @@
|
|||
#n:school
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
#n:information_schema
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
#n:pg_catalog
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="RemoteRepositoriesConfiguration">
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Central Repository" />
|
||||
<option name="url" value="https://repo.maven.apache.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="Apache" />
|
||||
<option name="name" value="Apache" />
|
||||
<option name="url" value="https://repository.apache.org/snapshots/" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Maven Central repository" />
|
||||
<option name="url" value="https://repo1.maven.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="jboss.community" />
|
||||
<option name="name" value="JBoss Community repository" />
|
||||
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
|
||||
</remote-repository>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
<component name="libraryTable">
|
||||
<library name="opengauss-jdbc-2.0.0">
|
||||
<CLASSES>
|
||||
<root url="jar://$PROJECT_DIR$/libs/opengauss-jdbc-2.0.0.jar!/" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</component>
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="MavenProjectsManager">
|
||||
<option name="originalFiles">
|
||||
<list>
|
||||
<option value="$PROJECT_DIR$/pom.xml" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Palette2">
|
||||
<group name="Swing">
|
||||
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
|
||||
</item>
|
||||
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
|
||||
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
|
||||
<initial-values>
|
||||
<property name="text" value="Button" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="RadioButton" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="CheckBox" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="Label" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
|
||||
<preferred-size width="-1" height="20" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
|
||||
</item>
|
||||
</group>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -1,18 +1,26 @@
|
|||
# SparkOpOpenGauss
|
||||
|
||||
这是将 openGuass 作为数据源的spark示例代码。
|
||||
|
||||
**环境和软件版本要求（前置要求）：**
|
||||
这是将 openGuass 作为数据源的spark示例代码，./src/main/scala/org.openguass.spark.sources.datasourcev2
|
||||
|
||||
环境和软件版本要求（前置要求）
|
||||
1. Java 1.8
|
||||
2. Scala 2.12
|
||||
3. postgresql.jar 或 opengauss-jdbc--${version}.jar(自己打包或官方提供的jar包)
|
||||
|
||||
运行说明：
|
||||
1. 请确保服务器上的数据库正常运行，且你的机器可正常连接数据库。
|
||||
2. 以sparkuser用户身份执行./resources/school.sql文件。
|
||||
3. 请修改代码中sparkuser用户对应的密码、连接数据库的ip及端口，即修改 x.x.x.x:port 。主要代码内容在 `./src/main/scala/`。
|
||||
4. 关于代码的构建
|
||||
+ 首先需要获取 openGauss JDBC 的 jar 包，放入本项目的根目录下的libs目录下,在IDEA中添加jar包为依赖。
|
||||
+ 其次本项目为 maven 项目，IDEA 会通过 maven自动下载其余依赖。
|
||||
5. 可以在IDEA中进入以下示例文件中的方法，单击右键可直接运行本示例。
|
||||
+ 可运行`./src/test/scala/org/opengauss/spark/OpenGaussExample`。
|
||||
+ 另一个直接使用Spark JDBC的例子是 `./src/main/scala/SQLDataSourceExample`。
|
||||
|
||||
**说明：**
|
||||
代码结构说明：
|
||||
1. `./src/main/scala/org.opengauss.spark.sources.datasourcev2`包下分别包含了实现Spark Datasource V2 接口的简单数据源的实现示例、简单多数据源的实现示例、流的实现与测试示例与流和批的实现与测试示例。
|
||||
2. `./src/main/scala/org.opengauss.spark.sources.opengauss`包下包含了将openGauss作为数据源实现Spark Datasouce V2的实现示例。
|
||||
3. `./src/test/scala/org/opengauss/spark/OpenGaussExample`包含了三个测试方法，分别是 简单数据源的使用与测试示例，读和写 openGauss的使用与测试示例。
|
||||
|
||||
1. 请确保服务器上的数据库正常运行，且你的机器可正常连接数据库
|
||||
2. 以sparkuser用户身份执行./resources/school.sql文件
|
||||
3. 请修改代码中连接数据库的ip及端口，即修改 x.x.x.x:port 。主要代码内容在 ./src/main/scala/
|
||||
4. 可以在idea运行本示例。
|
||||
+ 可运行./src/test/scala/org/opengauss/spark/OpenGaussExample。
|
||||
+ 另一个直接使用Spark JDBC的例子是 ./src/main/scala/SQLDataSourceExample。
|
||||
PS: Windows 系统在IDEA中调试 `DataSourceV2StreamAndBatchExample` 与 `DataSourceV2StreamingExample`可能会报错，请从[该链接](https://github.com/steveloughran/winutils/tree/master/hadoop-3.0.0/bin) 中下载 hadoop.dll 和 winutils.exe 文件，并将文件路径添加到环境变量中。
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
Binary file not shown.
|
|
@ -57,25 +57,13 @@
|
|||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.12</artifactId>
|
||||
<version>3.3.0-SNAPSHOT</version>
|
||||
<!-- <scope>provided</scope>-->
|
||||
</dependency>
|
||||
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
|
||||
<!-- https://mvnrepository.com/artifact/org.opengauss/opengauss-jdbc -->
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>org.postgresql</groupId>-->
|
||||
<!-- <artifactId>postgresql</artifactId>-->
|
||||
<!--<!– <version>42.2.23</version>–>-->
|
||||
<!-- <version>system</version>-->
|
||||
<!-- <systemPath>${project.basedir}/libs/opengauss-jdbc-2.0.0.jar</systemPath>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>org.postgresql</groupId>-->
|
||||
<!--<!– <artifactId></artifactId>–>-->
|
||||
<!--<!– <version>42.2.23</version>–>-->
|
||||
<!-- <version>system</version>-->
|
||||
<!-- <systemPath>${project.basedir}/libs/opengauss-jdbc-2.0.0.jar</systemPath>-->
|
||||
<!-- <groupId>org.opengauss</groupId>-->
|
||||
<!-- <artifactId>opengauss-jdbc</artifactId>-->
|
||||
<!-- <version>2.0.1-compatibility</version>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
</dependencies>
|
||||
|
|
|
|||
|
|
@ -1,102 +0,0 @@
|
|||
age,workclass,fnlwgt,education,education_num,martial_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,salary
|
||||
39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White, Male, 2174, 0, 40, United-States, <=50K
|
||||
50, Self-emp-not-inc, 83311, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 13, United-States, <=50K
|
||||
38, Private, 215646, HS-grad, 9, Divorced, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 234721, 11th, 7, Married-civ-spouse, Handlers-cleaners, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 338409, Bachelors, 13, Married-civ-spouse, Prof-specialty, Wife, Black, Female, 0, 0, 40, Cuba, <=50K
|
||||
37, Private, 284582, Masters, 14, Married-civ-spouse, Exec-managerial, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
49, Private, 160187, 9th, 5, Married-spouse-absent, Other-service, Not-in-family, Black, Female, 0, 0, 16, Jamaica, <=50K
|
||||
52, Self-emp-not-inc, 209642, HS-grad, 9, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
31, Private, 45781, Masters, 14, Never-married, Prof-specialty, Not-in-family, White, Female, 14084, 0, 50, United-States, >50K
|
||||
42, Private, 159449, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 5178, 0, 40, United-States, >50K
|
||||
37, Private, 280464, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, Black, Male, 0, 0, 80, United-States, >50K
|
||||
30, State-gov, 141297, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Asian-Pac-Islander, Male, 0, 0, 40, India, >50K
|
||||
23, Private, 122272, Bachelors, 13, Never-married, Adm-clerical, Own-child, White, Female, 0, 0, 30, United-States, <=50K
|
||||
32, Private, 205019, Assoc-acdm, 12, Never-married, Sales, Not-in-family, Black, Male, 0, 0, 50, United-States, <=50K
|
||||
40, Private, 121772, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male, 0, 0, 40, ?, >50K
|
||||
34, Private, 245487, 7th-8th, 4, Married-civ-spouse, Transport-moving, Husband, Amer-Indian-Eskimo, Male, 0, 0, 45, Mexico, <=50K
|
||||
25, Self-emp-not-inc, 176756, HS-grad, 9, Never-married, Farming-fishing, Own-child, White, Male, 0, 0, 35, United-States, <=50K
|
||||
32, Private, 186824, HS-grad, 9, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, United-States, <=50K
|
||||
38, Private, 28887, 11th, 7, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
43, Self-emp-not-inc, 292175, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 45, United-States, >50K
|
||||
40, Private, 193524, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 60, United-States, >50K
|
||||
54, Private, 302146, HS-grad, 9, Separated, Other-service, Unmarried, Black, Female, 0, 0, 20, United-States, <=50K
|
||||
35, Federal-gov, 76845, 9th, 5, Married-civ-spouse, Farming-fishing, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Private, 117037, 11th, 7, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 2042, 40, United-States, <=50K
|
||||
59, Private, 109015, HS-grad, 9, Divorced, Tech-support, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
56, Local-gov, 216851, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
19, Private, 168294, HS-grad, 9, Never-married, Craft-repair, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
54, ?, 180211, Some-college, 10, Married-civ-spouse, ?, Husband, Asian-Pac-Islander, Male, 0, 0, 60, South, >50K
|
||||
39, Private, 367260, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 80, United-States, <=50K
|
||||
49, Private, 193366, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
23, Local-gov, 190709, Assoc-acdm, 12, Never-married, Protective-serv, Not-in-family, White, Male, 0, 0, 52, United-States, <=50K
|
||||
20, Private, 266015, Some-college, 10, Never-married, Sales, Own-child, Black, Male, 0, 0, 44, United-States, <=50K
|
||||
45, Private, 386940, Bachelors, 13, Divorced, Exec-managerial, Own-child, White, Male, 0, 1408, 40, United-States, <=50K
|
||||
30, Federal-gov, 59951, Some-college, 10, Married-civ-spouse, Adm-clerical, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
22, State-gov, 311512, Some-college, 10, Married-civ-spouse, Other-service, Husband, Black, Male, 0, 0, 15, United-States, <=50K
|
||||
48, Private, 242406, 11th, 7, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
21, Private, 197200, Some-college, 10, Never-married, Machine-op-inspct, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 544091, HS-grad, 9, Married-AF-spouse, Adm-clerical, Wife, White, Female, 0, 0, 25, United-States, <=50K
|
||||
31, Private, 84154, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 38, ?, >50K
|
||||
48, Self-emp-not-inc, 265477, Assoc-acdm, 12, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
31, Private, 507875, 9th, 5, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 43, United-States, <=50K
|
||||
53, Self-emp-not-inc, 88506, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
24, Private, 172987, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
49, Private, 94638, HS-grad, 9, Separated, Adm-clerical, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
25, Private, 289980, HS-grad, 9, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 35, United-States, <=50K
|
||||
57, Federal-gov, 337895, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Black, Male, 0, 0, 40, United-States, >50K
|
||||
53, Private, 144361, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 38, United-States, <=50K
|
||||
44, Private, 128354, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
41, State-gov, 101603, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
29, Private, 271466, Assoc-voc, 11, Never-married, Prof-specialty, Not-in-family, White, Male, 0, 0, 43, United-States, <=50K
|
||||
25, Private, 32275, Some-college, 10, Married-civ-spouse, Exec-managerial, Wife, Other, Female, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 226956, HS-grad, 9, Never-married, Other-service, Own-child, White, Female, 0, 0, 30, ?, <=50K
|
||||
47, Private, 51835, Prof-school, 15, Married-civ-spouse, Prof-specialty, Wife, White, Female, 0, 1902, 60, Honduras, >50K
|
||||
50, Federal-gov, 251585, Bachelors, 13, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 55, United-States, >50K
|
||||
47, Self-emp-inc, 109832, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 60, United-States, <=50K
|
||||
43, Private, 237993, Some-college, 10, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
46, Private, 216666, 5th-6th, 3, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
35, Private, 56352, Assoc-voc, 11, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
41, Private, 147372, HS-grad, 9, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
30, Private, 188146, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 5013, 0, 40, United-States, <=50K
|
||||
30, Private, 59496, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 2407, 0, 40, United-States, <=50K
|
||||
32, ?, 293936, 7th-8th, 4, Married-spouse-absent, ?, Not-in-family, White, Male, 0, 0, 40, ?, <=50K
|
||||
48, Private, 149640, HS-grad, 9, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
42, Private, 116632, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
29, Private, 105598, Some-college, 10, Divorced, Tech-support, Not-in-family, White, Male, 0, 0, 58, United-States, <=50K
|
||||
36, Private, 155537, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 183175, Some-college, 10, Divorced, Adm-clerical, Not-in-family, White, Female, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 169846, HS-grad, 9, Married-civ-spouse, Adm-clerical, Wife, White, Female, 0, 0, 40, United-States, >50K
|
||||
49, Self-emp-inc, 191681, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 50, United-States, >50K
|
||||
25, ?, 200681, Some-college, 10, Never-married, ?, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 101509, Some-college, 10, Never-married, Prof-specialty, Own-child, White, Male, 0, 0, 32, United-States, <=50K
|
||||
31, Private, 309974, Bachelors, 13, Separated, Sales, Own-child, Black, Female, 0, 0, 40, United-States, <=50K
|
||||
29, Self-emp-not-inc, 162298, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 70, United-States, >50K
|
||||
23, Private, 211678, Some-college, 10, Never-married, Machine-op-inspct, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
79, Private, 124744, Some-college, 10, Married-civ-spouse, Prof-specialty, Other-relative, White, Male, 0, 0, 20, United-States, <=50K
|
||||
27, Private, 213921, HS-grad, 9, Never-married, Other-service, Own-child, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
40, Private, 32214, Assoc-acdm, 12, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
67, ?, 212759, 10th, 6, Married-civ-spouse, ?, Husband, White, Male, 0, 0, 2, United-States, <=50K
|
||||
18, Private, 309634, 11th, 7, Never-married, Other-service, Own-child, White, Female, 0, 0, 22, United-States, <=50K
|
||||
31, Local-gov, 125927, 7th-8th, 4, Married-civ-spouse, Farming-fishing, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 446839, HS-grad, 9, Never-married, Sales, Not-in-family, White, Male, 0, 0, 30, United-States, <=50K
|
||||
52, Private, 276515, Bachelors, 13, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Cuba, <=50K
|
||||
46, Private, 51618, HS-grad, 9, Married-civ-spouse, Other-service, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
59, Private, 159937, HS-grad, 9, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
44, Private, 343591, HS-grad, 9, Divorced, Craft-repair, Not-in-family, White, Female, 14344, 0, 40, United-States, >50K
|
||||
53, Private, 346253, HS-grad, 9, Divorced, Sales, Own-child, White, Female, 0, 0, 35, United-States, <=50K
|
||||
49, Local-gov, 268234, HS-grad, 9, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
33, Private, 202051, Masters, 14, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
30, Private, 54334, 9th, 5, Never-married, Sales, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Federal-gov, 410867, Doctorate, 16, Never-married, Prof-specialty, Not-in-family, White, Female, 0, 0, 50, United-States, >50K
|
||||
57, Private, 249977, Assoc-voc, 11, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
37, Private, 286730, Some-college, 10, Divorced, Craft-repair, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 212563, Some-college, 10, Divorced, Machine-op-inspct, Unmarried, Black, Female, 0, 0, 25, United-States, <=50K
|
||||
30, Private, 117747, HS-grad, 9, Married-civ-spouse, Sales, Wife, Asian-Pac-Islander, Female, 0, 1573, 35, ?, <=50K
|
||||
34, Local-gov, 226296, Bachelors, 13, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
29, Local-gov, 115585, Some-college, 10, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 50, United-States, <=50K
|
||||
48, Self-emp-not-inc, 191277, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 1902, 60, United-States, >50K
|
||||
37, Private, 202683, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, >50K
|
||||
48, Private, 171095, Assoc-acdm, 12, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, England, <=50K
|
||||
32, Federal-gov, 249409, HS-grad, 9, Never-married, Other-service, Own-child, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
76, Private, 124191, Masters, 14, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
|
|
|
@ -1,5 +0,0 @@
|
|||
customerId,customerName
|
||||
1,John
|
||||
2,Clerk
|
||||
3,Micheal
|
||||
4,Sample
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -1,15 +0,0 @@
|
|||
transactionId,customerId,itemId,amountPaid
|
||||
111,1,1,100.0
|
||||
112,2,2,505.0
|
||||
113,3,3,510.0
|
||||
114,4,4,600.0
|
||||
115,1,2,500.0
|
||||
116,1,2,500.0
|
||||
117,1,2,500.0
|
||||
118,1,2,500.0
|
||||
119,2,3,500.0
|
||||
120,1,2,500.0
|
||||
121,1,4,500.0
|
||||
122,1,2,500.0
|
||||
123,1,4,500.0
|
||||
124,1,2,500.0
|
||||
|
|
|
@ -16,385 +16,55 @@ object SQLDataSourceExample {
|
|||
.config("spark.some.config.option", "some-value")
|
||||
.getOrCreate()
|
||||
|
||||
// runBasicDataSourceExample(spark)
|
||||
// runGenericFileSourceOptionsExample(spark)
|
||||
// runBasicParquetExample(spark)
|
||||
// runParquetSchemaMergingExample(spark)
|
||||
// runJsonDatasetExample(spark)
|
||||
// runCsvDatasetExample(spark)
|
||||
// runTextDatasetExample(spark)
|
||||
runJdbcDatasetExample(spark)
|
||||
|
||||
spark.stop()
|
||||
}
|
||||
|
||||
private def runGenericFileSourceOptionsExample(spark: SparkSession): Unit = {
|
||||
// $example on:ignore_corrupt_files$
|
||||
// enable ignore corrupt files
|
||||
spark.sql("set spark.sql.files.ignoreCorruptFiles=true")
|
||||
// dir1/file3.json is corrupt from parquet's view
|
||||
val testCorruptDF = spark.read.parquet(
|
||||
"examples/src/main/resources/dir1/",
|
||||
"examples/src/main/resources/dir1/dir2/")
|
||||
testCorruptDF.show()
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// |file2.parquet|
|
||||
// +-------------+
|
||||
// $example off:ignore_corrupt_files$
|
||||
// $example on:recursive_file_lookup$
|
||||
val recursiveLoadedDF = spark.read.format("parquet")
|
||||
.option("recursiveFileLookup", "true")
|
||||
.load("examples/src/main/resources/dir1")
|
||||
recursiveLoadedDF.show()
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// |file2.parquet|
|
||||
// +-------------+
|
||||
// $example off:recursive_file_lookup$
|
||||
spark.sql("set spark.sql.files.ignoreCorruptFiles=false")
|
||||
// $example on:load_with_path_glob_filter$
|
||||
val testGlobFilterDF = spark.read.format("parquet")
|
||||
.option("pathGlobFilter", "*.parquet") // json file should be filtered out
|
||||
.load("examples/src/main/resources/dir1")
|
||||
testGlobFilterDF.show()
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// +-------------+
|
||||
// $example off:load_with_path_glob_filter$
|
||||
// $example on:load_with_modified_time_filter$
|
||||
val beforeFilterDF = spark.read.format("parquet")
|
||||
// Files modified before 07/01/2020 at 05:30 are allowed
|
||||
.option("modifiedBefore", "2020-07-01T05:30:00")
|
||||
.load("examples/src/main/resources/dir1");
|
||||
beforeFilterDF.show();
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// +-------------+
|
||||
val afterFilterDF = spark.read.format("parquet")
|
||||
// Files modified after 06/01/2020 at 05:30 are allowed
|
||||
.option("modifiedAfter", "2020-06-01T05:30:00")
|
||||
.load("examples/src/main/resources/dir1");
|
||||
afterFilterDF.show();
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// +-------------+
|
||||
// $example off:load_with_modified_time_filter$
|
||||
}
|
||||
|
||||
private def runBasicDataSourceExample(spark: SparkSession): Unit = {
|
||||
// $example on:generic_load_save_functions$
|
||||
val usersDF = spark.read.load("examples/src/main/resources/users.parquet")
|
||||
usersDF.select("name", "favorite_color").write.save("namesAndFavColors.parquet")
|
||||
// $example off:generic_load_save_functions$
|
||||
// $example on:manual_load_options$
|
||||
val peopleDF = spark.read.format("json").load("examples/src/main/resources/people.json")
|
||||
peopleDF.select("name", "age").write.format("parquet").save("namesAndAges.parquet")
|
||||
// $example off:manual_load_options$
|
||||
// $example on:manual_load_options_csv$
|
||||
val peopleDFCsv = spark.read.format("csv")
|
||||
.option("sep", ";")
|
||||
.option("inferSchema", "true")
|
||||
.option("header", "true")
|
||||
.load("examples/src/main/resources/people.csv")
|
||||
// $example off:manual_load_options_csv$
|
||||
// $example on:manual_save_options_orc$
|
||||
usersDF.write.format("orc")
|
||||
.option("orc.bloom.filter.columns", "favorite_color")
|
||||
.option("orc.dictionary.key.threshold", "1.0")
|
||||
.option("orc.column.encoding.direct", "name")
|
||||
.save("users_with_options.orc")
|
||||
// $example off:manual_save_options_orc$
|
||||
// $example on:manual_save_options_parquet$
|
||||
usersDF.write.format("parquet")
|
||||
.option("parquet.bloom.filter.enabled#favorite_color", "true")
|
||||
.option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000")
|
||||
.option("parquet.enable.dictionary", "true")
|
||||
.option("parquet.page.write-checksum.enabled", "false")
|
||||
.save("users_with_options.parquet")
|
||||
// $example off:manual_save_options_parquet$
|
||||
|
||||
// $example on:direct_sql$
|
||||
val sqlDF = spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`")
|
||||
// $example off:direct_sql$
|
||||
// $example on:write_sorting_and_bucketing$
|
||||
peopleDF.write.bucketBy(42, "name").sortBy("age").saveAsTable("people_bucketed")
|
||||
// $example off:write_sorting_and_bucketing$
|
||||
// $example on:write_partitioning$
|
||||
usersDF.write.partitionBy("favorite_color").format("parquet").save("namesPartByColor.parquet")
|
||||
// $example off:write_partitioning$
|
||||
// $example on:write_partition_and_bucket$
|
||||
usersDF
|
||||
.write
|
||||
.partitionBy("favorite_color")
|
||||
.bucketBy(42, "name")
|
||||
.saveAsTable("users_partitioned_bucketed")
|
||||
// $example off:write_partition_and_bucket$
|
||||
|
||||
spark.sql("DROP TABLE IF EXISTS people_bucketed")
|
||||
spark.sql("DROP TABLE IF EXISTS users_partitioned_bucketed")
|
||||
}
|
||||
|
||||
private def runBasicParquetExample(spark: SparkSession): Unit = {
|
||||
// $example on:basic_parquet_example$
|
||||
// Encoders for most common types are automatically provided by importing spark.implicits._
|
||||
import spark.implicits._
|
||||
|
||||
val peopleDF = spark.read.json("examples/src/main/resources/people.json")
|
||||
|
||||
// DataFrames can be saved as Parquet files, maintaining the schema information
|
||||
peopleDF.write.parquet("people.parquet")
|
||||
|
||||
// Read in the parquet file created above
|
||||
// Parquet files are self-describing so the schema is preserved
|
||||
// The result of loading a Parquet file is also a DataFrame
|
||||
val parquetFileDF = spark.read.parquet("people.parquet")
|
||||
|
||||
// Parquet files can also be used to create a temporary view and then used in SQL statements
|
||||
parquetFileDF.createOrReplaceTempView("parquetFile")
|
||||
val namesDF = spark.sql("SELECT name FROM parquetFile WHERE age BETWEEN 13 AND 19")
|
||||
namesDF.map(attributes => "Name: " + attributes(0)).show()
|
||||
// +------------+
|
||||
// | value|
|
||||
// +------------+
|
||||
// |Name: Justin|
|
||||
// +------------+
|
||||
// $example off:basic_parquet_example$
|
||||
}
|
||||
|
||||
private def runParquetSchemaMergingExample(spark: SparkSession): Unit = {
|
||||
// $example on:schema_merging$
|
||||
// This is used to implicitly convert an RDD to a DataFrame.
|
||||
import spark.implicits._
|
||||
|
||||
// Create a simple DataFrame, store into a partition directory
|
||||
val squaresDF = spark.sparkContext.makeRDD(1 to 5).map(i => (i, i * i)).toDF("value", "square")
|
||||
squaresDF.write.parquet("data/test_table/key=1")
|
||||
|
||||
// Create another DataFrame in a new partition directory,
|
||||
// adding a new column and dropping an existing column
|
||||
val cubesDF = spark.sparkContext.makeRDD(6 to 10).map(i => (i, i * i * i)).toDF("value", "cube")
|
||||
cubesDF.write.parquet("data/test_table/key=2")
|
||||
|
||||
// Read the partitioned table
|
||||
val mergedDF = spark.read.option("mergeSchema", "true").parquet("data/test_table")
|
||||
mergedDF.printSchema()
|
||||
|
||||
// The final schema consists of all 3 columns in the Parquet files together
|
||||
// with the partitioning column appeared in the partition directory paths
|
||||
// root
|
||||
// |-- value: int (nullable = true)
|
||||
// |-- square: int (nullable = true)
|
||||
// |-- cube: int (nullable = true)
|
||||
// |-- key: int (nullable = true)
|
||||
// $example off:schema_merging$
|
||||
}
|
||||
|
||||
private def runJsonDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:json_dataset$
|
||||
// Primitive types (Int, String, etc) and Product types (case classes) encoders are
|
||||
// supported by importing this when creating a Dataset.
|
||||
import spark.implicits._
|
||||
|
||||
// A JSON dataset is pointed to by path.
|
||||
// The path can be either a single text file or a directory storing text files
|
||||
val path = "examples/src/main/resources/people.json"
|
||||
val peopleDF = spark.read.json(path)
|
||||
|
||||
// The inferred schema can be visualized using the printSchema() method
|
||||
peopleDF.printSchema()
|
||||
// root
|
||||
// |-- age: long (nullable = true)
|
||||
// |-- name: string (nullable = true)
|
||||
|
||||
// Creates a temporary view using the DataFrame
|
||||
peopleDF.createOrReplaceTempView("people")
|
||||
|
||||
// SQL statements can be run by using the sql methods provided by spark
|
||||
val teenagerNamesDF = spark.sql("SELECT name FROM people WHERE age BETWEEN 13 AND 19")
|
||||
teenagerNamesDF.show()
|
||||
// +------+
|
||||
// | name|
|
||||
// +------+
|
||||
// |Justin|
|
||||
// +------+
|
||||
|
||||
// Alternatively, a DataFrame can be created for a JSON dataset represented by
|
||||
// a Dataset[String] storing one JSON object per string
|
||||
val otherPeopleDataset = spark.createDataset(
|
||||
"""{"name":"Yin","address":{"city":"Columbus","state":"Ohio"}}""" :: Nil)
|
||||
val otherPeople = spark.read.json(otherPeopleDataset)
|
||||
otherPeople.show()
|
||||
// +---------------+----+
|
||||
// | address|name|
|
||||
// +---------------+----+
|
||||
// |[Columbus,Ohio]| Yin|
|
||||
// +---------------+----+
|
||||
// $example off:json_dataset$
|
||||
}
|
||||
|
||||
private def runCsvDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:csv_dataset$
|
||||
// A CSV dataset is pointed to by path.
|
||||
// The path can be either a single CSV file or a directory of CSV files
|
||||
val path = "examples/src/main/resources/people.csv"
|
||||
|
||||
val df = spark.read.csv(path)
|
||||
df.show()
|
||||
// +------------------+
|
||||
// | _c0|
|
||||
// +------------------+
|
||||
// | name;age;job|
|
||||
// |Jorge;30;Developer|
|
||||
// | Bob;32;Developer|
|
||||
// +------------------+
|
||||
|
||||
// Read a csv with delimiter, the default delimiter is ","
|
||||
val df2 = spark.read.option("delimiter", ";").csv(path)
|
||||
df2.show()
|
||||
// +-----+---+---------+
|
||||
// | _c0|_c1| _c2|
|
||||
// +-----+---+---------+
|
||||
// | name|age| job|
|
||||
// |Jorge| 30|Developer|
|
||||
// | Bob| 32|Developer|
|
||||
// +-----+---+---------+
|
||||
|
||||
// Read a csv with delimiter and a header
|
||||
val df3 = spark.read.option("delimiter", ";").option("header", "true").csv(path)
|
||||
df3.show()
|
||||
// +-----+---+---------+
|
||||
// | name|age| job|
|
||||
// +-----+---+---------+
|
||||
// |Jorge| 30|Developer|
|
||||
// | Bob| 32|Developer|
|
||||
// +-----+---+---------+
|
||||
|
||||
// You can also use options() to use multiple options
|
||||
val df4 = spark.read.options(Map("delimiter" -> ";", "header" -> "true")).csv(path)
|
||||
|
||||
// "output" is a folder which contains multiple csv files and a _SUCCESS file.
|
||||
df3.write.csv("output")
|
||||
|
||||
// Read all files in a folder, please make sure only CSV files should present in the folder.
|
||||
val folderPath = "examples/src/main/resources";
|
||||
val df5 = spark.read.csv(folderPath);
|
||||
df5.show();
|
||||
// Wrong schema because non-CSV files are read
|
||||
// +-----------+
|
||||
// | _c0|
|
||||
// +-----------+
|
||||
// |238val_238|
|
||||
// | 86val_86|
|
||||
// |311val_311|
|
||||
// | 27val_27|
|
||||
// |165val_165|
|
||||
// +-----------+
|
||||
|
||||
// $example off:csv_dataset$
|
||||
}
|
||||
|
||||
private def runTextDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:text_dataset$
|
||||
// A text dataset is pointed to by path.
|
||||
// The path can be either a single text file or a directory of text files
|
||||
val path = "examples/src/main/resources/people.txt"
|
||||
|
||||
val df1 = spark.read.text(path)
|
||||
df1.show()
|
||||
// +-----------+
|
||||
// | value|
|
||||
// +-----------+
|
||||
// |Michael, 29|
|
||||
// | Andy, 30|
|
||||
// | Justin, 19|
|
||||
// +-----------+
|
||||
|
||||
// You can use 'lineSep' option to define the line separator.
|
||||
// The line separator handles all `\r`, `\r\n` and `\n` by default.
|
||||
val df2 = spark.read.option("lineSep", ",").text(path)
|
||||
df2.show()
|
||||
// +-----------+
|
||||
// | value|
|
||||
// +-----------+
|
||||
// | Michael|
|
||||
// | 29\nAndy|
|
||||
// | 30\nJustin|
|
||||
// | 19\n|
|
||||
// +-----------+
|
||||
|
||||
// You can also use 'wholetext' option to read each input file as a single row.
|
||||
val df3 = spark.read.option("wholetext", true).text(path)
|
||||
df3.show()
|
||||
// +--------------------+
|
||||
// | value|
|
||||
// +--------------------+
|
||||
// |Michael, 29\nAndy...|
|
||||
// +--------------------+
|
||||
|
||||
// "output" is a folder which contains multiple text files and a _SUCCESS file.
|
||||
df1.write.text("output")
|
||||
|
||||
// You can specify the compression format using the 'compression' option.
|
||||
df1.write.option("compression", "gzip").text("output_compressed")
|
||||
|
||||
// $example off:text_dataset$
|
||||
}
|
||||
|
||||
private def runJdbcDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:jdbc_dataset$
|
||||
// Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
|
||||
// Loading data from a JDBC source
|
||||
val dburl = "jdbc:postgresql://x.x.x.x:port/school"
|
||||
val jdbcDF = spark.read
|
||||
.format("jdbc")
|
||||
.option("url", "jdbc:postgresql://x.x.x.x:port/school")
|
||||
.option("url", dburl)
|
||||
.option("dbtable", "class")
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "Enmo@123")
|
||||
.option("password", "pwdofsparkuser")
|
||||
.load()
|
||||
//.show()
|
||||
|
||||
val connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "sparkuser")
|
||||
connectionProperties.put("password", "Enmo@123")
|
||||
connectionProperties.put("password", "pwdofsparkuser")
|
||||
val jdbcDF2 = spark.read
|
||||
.option("customSchema","cla_id INT")
|
||||
.jdbc("jdbc:postgresql://x.x.x.x:port/school", "class", connectionProperties).show()
|
||||
.option("customSchema","cla_id INT, cla_teacher STRING")
|
||||
.jdbc(dburl, "class", connectionProperties)
|
||||
// Specifying the custom data types of the read schema
|
||||
|
||||
|
||||
connectionProperties.put("customSchema", "cla_id INT")//, cla_name STRING")
|
||||
connectionProperties.put("customSchema", "cla_id INT, cla_name STRING")
|
||||
val jdbcDF3 = spark.read
|
||||
.jdbc("jdbc:postgresql://x.x.x.x:port/school", "class", connectionProperties)
|
||||
.jdbc(dburl, "class", connectionProperties)
|
||||
|
||||
// // Saving data to a JDBC source. Create table "customtable1", and write data
|
||||
// jdbcDF.write
|
||||
// .format("jdbc")
|
||||
// .option("url", "jdbc:postgresql://x.x.x.x:port/school")
|
||||
// .option("dbtable", "customtable1")
|
||||
// .option("user", "sparkuser")
|
||||
// .option("password", "Enmo@123")
|
||||
// .save()
|
||||
//
|
||||
//
|
||||
// jdbcDF2.write
|
||||
// .jdbc("jdbc:postgresql://x.x.x.x:port/school", "customtable2", connectionProperties)
|
||||
// Saving data to a JDBC source. Create table "customtable1", and write data
|
||||
jdbcDF.write
|
||||
.format("jdbc")
|
||||
.option("url", dburl)
|
||||
.option("dbtable", "customtable1")
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "pwdofsparkuser")
|
||||
.save()
|
||||
|
||||
|
||||
jdbcDF2.write
|
||||
.jdbc(dburl, "customtable2", connectionProperties)
|
||||
|
||||
// Specifying create table column data types on write
|
||||
jdbcDF3.show()
|
||||
|
||||
// jdbcDF3.write
|
||||
// .option("createTableColumnTypes", "cla_id INT, cla_name VARCHAR(20)")
|
||||
// .jdbc("jdbc:postgresql://x.x.x.x:port/school", "customtable3", connectionProperties)
|
||||
// // $example off:jdbc_dataset$
|
||||
jdbcDF3.write
|
||||
.option("createTableColumnTypes", "cla_id INT, cla_name VARCHAR(20)")
|
||||
.jdbc(dburl, "customtable3", connectionProperties)
|
||||
// $example off:jdbc_dataset$
|
||||
}
|
||||
}
|
||||
|
|
@ -15,10 +15,10 @@ import scala.collection.JavaConverters._
|
|||
/*
|
||||
* Default source should some kind of relation provider
|
||||
*/
|
||||
class DefaultSource extends TableProvider{
|
||||
class DefaultSource extends TableProvider {
|
||||
|
||||
override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType =
|
||||
getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema()
|
||||
override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType =
|
||||
getTable(null, Array.empty[Transform], caseInsensitiveStringMap.asCaseSensitiveMap()).schema()
|
||||
|
||||
override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table =
|
||||
new SimpleBatchTable()
|
||||
|
|
@ -54,20 +54,21 @@ class SimpleScanBuilder extends ScanBuilder {
|
|||
The schema is repeated here as it can change after column pruning etc
|
||||
*/
|
||||
|
||||
class SimpleScan extends Scan with Batch{
|
||||
override def readSchema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
class SimpleScan extends Scan with Batch {
|
||||
override def readSchema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def toBatch: Batch = this
|
||||
|
||||
override def planInputPartitions(): Array[InputPartition] = {
|
||||
Array(new SimplePartition(0,4),
|
||||
new SimplePartition(5,9))
|
||||
Array(new SimplePartition(0, 4),
|
||||
new SimplePartition(5, 9))
|
||||
}
|
||||
|
||||
override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory()
|
||||
}
|
||||
|
||||
// simple class to organise the partition
|
||||
class SimplePartition(val start:Int, val end:Int) extends InputPartition
|
||||
class SimplePartition(val start: Int, val end: Int) extends InputPartition
|
||||
|
||||
// reader factory
|
||||
class SimplePartitionReaderFactory extends PartitionReaderFactory {
|
||||
|
|
@ -79,7 +80,7 @@ class SimplePartitionReaderFactory extends PartitionReaderFactory {
|
|||
// parathion reader
|
||||
class SimplePartitionReader(inputPartition: SimplePartition) extends PartitionReader[InternalRow] {
|
||||
|
||||
val values = Array("1", "2", "3", "4", "5","6","7","8","9","10")
|
||||
val values = Array("1", "2", "3", "4", "5", "6", "7", "8", "9", "10")
|
||||
|
||||
var index = inputPartition.start
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ object DataSourceV2StreamAndBatchExample {
|
|||
.getOrCreate()
|
||||
|
||||
|
||||
val dataSource = "cn.ecnu.spark.sources.datasourcev2.streamandbatch.simple"
|
||||
val dataSource = "org.opengauss.spark.sources.datasourcev2.streamandbatch"
|
||||
|
||||
val batchDf = sparkSession
|
||||
.read
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ object DataSourceV2StreamingExample {
|
|||
|
||||
val streamingDf = sparkSession.
|
||||
readStream.
|
||||
format("cn.ecnu.spark.sources.datasourcev2.streaming.simple")
|
||||
format("org.opengauss.spark.sources.datasourcev2.streaming")
|
||||
.load()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import org.scalatest.Matchers.convertToAnyShouldWrapper
|
|||
class OpenGaussExample extends FlatSpec {
|
||||
|
||||
val testTableName = "course"
|
||||
val dburl = "jdbc:postgresql://x.x.x.x:port/school"
|
||||
|
||||
"Simple data source" should "read" in{
|
||||
val sparkSession = SparkSession.builder
|
||||
|
|
@ -18,7 +19,7 @@ class OpenGaussExample extends FlatSpec {
|
|||
.getOrCreate()
|
||||
|
||||
val simpleDf = sparkSession.read
|
||||
.format("cn.ecnu.spark.sources.datasourcev2.simple")
|
||||
.format("org.opengauss.spark.sources.datasourcev2.simple")
|
||||
.load()
|
||||
|
||||
simpleDf.show()
|
||||
|
|
@ -37,9 +38,9 @@ class OpenGaussExample extends FlatSpec {
|
|||
val simpleRead = spark
|
||||
.read
|
||||
.format("org.opengauss.spark.sources.opengauss")
|
||||
.option("url", "jdbc:postgresql://x.x.x.x:port/school")
|
||||
.option("url", dburl)
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "Enmo@123")
|
||||
.option("password", "pwdofsparkuser")
|
||||
.option("tableName", testTableName)
|
||||
.option("partitionSize", 10)
|
||||
// .option("partitionColumn", "name")
|
||||
|
|
@ -63,9 +64,9 @@ class OpenGaussExample extends FlatSpec {
|
|||
df
|
||||
.write
|
||||
.format("org.opengauss.spark.sources.opengauss")
|
||||
.option("url", "jdbc:postgresql://x.x.x.x:port/postgres")
|
||||
.option("url", dburl)
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "Enmo@123")
|
||||
.option("password", "pwdofsparkuser")
|
||||
.option("tableName", testTableName)
|
||||
.option("partitionSize", 10)
|
||||
.option("partitionColumn", "product_no")
|
||||
|
|
@ -77,14 +78,6 @@ class OpenGaussExample extends FlatSpec {
|
|||
|
||||
|
||||
|
||||
// def connection(c: PostgreSQLContainer) = {
|
||||
// Class.forName(c.driverClassName)
|
||||
// val properties = new Properties()
|
||||
// properties.put("user", c.username)
|
||||
// properties.put("password", c.password)
|
||||
// DriverManager.getConnection(c.jdbcUrl, properties)
|
||||
// }
|
||||
|
||||
object Queries {
|
||||
lazy val createTableQuery = s"CREATE TABLE $testTableName (user_id BIGINT PRIMARY KEY);"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,102 +0,0 @@
|
|||
age,workclass,fnlwgt,education,education_num,martial_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,salary
|
||||
39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White, Male, 2174, 0, 40, United-States, <=50K
|
||||
50, Self-emp-not-inc, 83311, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 13, United-States, <=50K
|
||||
38, Private, 215646, HS-grad, 9, Divorced, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 234721, 11th, 7, Married-civ-spouse, Handlers-cleaners, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 338409, Bachelors, 13, Married-civ-spouse, Prof-specialty, Wife, Black, Female, 0, 0, 40, Cuba, <=50K
|
||||
37, Private, 284582, Masters, 14, Married-civ-spouse, Exec-managerial, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
49, Private, 160187, 9th, 5, Married-spouse-absent, Other-service, Not-in-family, Black, Female, 0, 0, 16, Jamaica, <=50K
|
||||
52, Self-emp-not-inc, 209642, HS-grad, 9, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
31, Private, 45781, Masters, 14, Never-married, Prof-specialty, Not-in-family, White, Female, 14084, 0, 50, United-States, >50K
|
||||
42, Private, 159449, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 5178, 0, 40, United-States, >50K
|
||||
37, Private, 280464, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, Black, Male, 0, 0, 80, United-States, >50K
|
||||
30, State-gov, 141297, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Asian-Pac-Islander, Male, 0, 0, 40, India, >50K
|
||||
23, Private, 122272, Bachelors, 13, Never-married, Adm-clerical, Own-child, White, Female, 0, 0, 30, United-States, <=50K
|
||||
32, Private, 205019, Assoc-acdm, 12, Never-married, Sales, Not-in-family, Black, Male, 0, 0, 50, United-States, <=50K
|
||||
40, Private, 121772, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male, 0, 0, 40, ?, >50K
|
||||
34, Private, 245487, 7th-8th, 4, Married-civ-spouse, Transport-moving, Husband, Amer-Indian-Eskimo, Male, 0, 0, 45, Mexico, <=50K
|
||||
25, Self-emp-not-inc, 176756, HS-grad, 9, Never-married, Farming-fishing, Own-child, White, Male, 0, 0, 35, United-States, <=50K
|
||||
32, Private, 186824, HS-grad, 9, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, United-States, <=50K
|
||||
38, Private, 28887, 11th, 7, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
43, Self-emp-not-inc, 292175, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 45, United-States, >50K
|
||||
40, Private, 193524, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 60, United-States, >50K
|
||||
54, Private, 302146, HS-grad, 9, Separated, Other-service, Unmarried, Black, Female, 0, 0, 20, United-States, <=50K
|
||||
35, Federal-gov, 76845, 9th, 5, Married-civ-spouse, Farming-fishing, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Private, 117037, 11th, 7, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 2042, 40, United-States, <=50K
|
||||
59, Private, 109015, HS-grad, 9, Divorced, Tech-support, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
56, Local-gov, 216851, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
19, Private, 168294, HS-grad, 9, Never-married, Craft-repair, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
54, ?, 180211, Some-college, 10, Married-civ-spouse, ?, Husband, Asian-Pac-Islander, Male, 0, 0, 60, South, >50K
|
||||
39, Private, 367260, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 80, United-States, <=50K
|
||||
49, Private, 193366, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
23, Local-gov, 190709, Assoc-acdm, 12, Never-married, Protective-serv, Not-in-family, White, Male, 0, 0, 52, United-States, <=50K
|
||||
20, Private, 266015, Some-college, 10, Never-married, Sales, Own-child, Black, Male, 0, 0, 44, United-States, <=50K
|
||||
45, Private, 386940, Bachelors, 13, Divorced, Exec-managerial, Own-child, White, Male, 0, 1408, 40, United-States, <=50K
|
||||
30, Federal-gov, 59951, Some-college, 10, Married-civ-spouse, Adm-clerical, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
22, State-gov, 311512, Some-college, 10, Married-civ-spouse, Other-service, Husband, Black, Male, 0, 0, 15, United-States, <=50K
|
||||
48, Private, 242406, 11th, 7, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
21, Private, 197200, Some-college, 10, Never-married, Machine-op-inspct, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 544091, HS-grad, 9, Married-AF-spouse, Adm-clerical, Wife, White, Female, 0, 0, 25, United-States, <=50K
|
||||
31, Private, 84154, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 38, ?, >50K
|
||||
48, Self-emp-not-inc, 265477, Assoc-acdm, 12, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
31, Private, 507875, 9th, 5, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 43, United-States, <=50K
|
||||
53, Self-emp-not-inc, 88506, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
24, Private, 172987, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
49, Private, 94638, HS-grad, 9, Separated, Adm-clerical, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
25, Private, 289980, HS-grad, 9, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 35, United-States, <=50K
|
||||
57, Federal-gov, 337895, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Black, Male, 0, 0, 40, United-States, >50K
|
||||
53, Private, 144361, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 38, United-States, <=50K
|
||||
44, Private, 128354, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
41, State-gov, 101603, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
29, Private, 271466, Assoc-voc, 11, Never-married, Prof-specialty, Not-in-family, White, Male, 0, 0, 43, United-States, <=50K
|
||||
25, Private, 32275, Some-college, 10, Married-civ-spouse, Exec-managerial, Wife, Other, Female, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 226956, HS-grad, 9, Never-married, Other-service, Own-child, White, Female, 0, 0, 30, ?, <=50K
|
||||
47, Private, 51835, Prof-school, 15, Married-civ-spouse, Prof-specialty, Wife, White, Female, 0, 1902, 60, Honduras, >50K
|
||||
50, Federal-gov, 251585, Bachelors, 13, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 55, United-States, >50K
|
||||
47, Self-emp-inc, 109832, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 60, United-States, <=50K
|
||||
43, Private, 237993, Some-college, 10, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
46, Private, 216666, 5th-6th, 3, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
35, Private, 56352, Assoc-voc, 11, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
41, Private, 147372, HS-grad, 9, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
30, Private, 188146, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 5013, 0, 40, United-States, <=50K
|
||||
30, Private, 59496, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 2407, 0, 40, United-States, <=50K
|
||||
32, ?, 293936, 7th-8th, 4, Married-spouse-absent, ?, Not-in-family, White, Male, 0, 0, 40, ?, <=50K
|
||||
48, Private, 149640, HS-grad, 9, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
42, Private, 116632, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
29, Private, 105598, Some-college, 10, Divorced, Tech-support, Not-in-family, White, Male, 0, 0, 58, United-States, <=50K
|
||||
36, Private, 155537, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 183175, Some-college, 10, Divorced, Adm-clerical, Not-in-family, White, Female, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 169846, HS-grad, 9, Married-civ-spouse, Adm-clerical, Wife, White, Female, 0, 0, 40, United-States, >50K
|
||||
49, Self-emp-inc, 191681, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 50, United-States, >50K
|
||||
25, ?, 200681, Some-college, 10, Never-married, ?, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 101509, Some-college, 10, Never-married, Prof-specialty, Own-child, White, Male, 0, 0, 32, United-States, <=50K
|
||||
31, Private, 309974, Bachelors, 13, Separated, Sales, Own-child, Black, Female, 0, 0, 40, United-States, <=50K
|
||||
29, Self-emp-not-inc, 162298, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 70, United-States, >50K
|
||||
23, Private, 211678, Some-college, 10, Never-married, Machine-op-inspct, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
79, Private, 124744, Some-college, 10, Married-civ-spouse, Prof-specialty, Other-relative, White, Male, 0, 0, 20, United-States, <=50K
|
||||
27, Private, 213921, HS-grad, 9, Never-married, Other-service, Own-child, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
40, Private, 32214, Assoc-acdm, 12, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
67, ?, 212759, 10th, 6, Married-civ-spouse, ?, Husband, White, Male, 0, 0, 2, United-States, <=50K
|
||||
18, Private, 309634, 11th, 7, Never-married, Other-service, Own-child, White, Female, 0, 0, 22, United-States, <=50K
|
||||
31, Local-gov, 125927, 7th-8th, 4, Married-civ-spouse, Farming-fishing, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 446839, HS-grad, 9, Never-married, Sales, Not-in-family, White, Male, 0, 0, 30, United-States, <=50K
|
||||
52, Private, 276515, Bachelors, 13, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Cuba, <=50K
|
||||
46, Private, 51618, HS-grad, 9, Married-civ-spouse, Other-service, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
59, Private, 159937, HS-grad, 9, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
44, Private, 343591, HS-grad, 9, Divorced, Craft-repair, Not-in-family, White, Female, 14344, 0, 40, United-States, >50K
|
||||
53, Private, 346253, HS-grad, 9, Divorced, Sales, Own-child, White, Female, 0, 0, 35, United-States, <=50K
|
||||
49, Local-gov, 268234, HS-grad, 9, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
33, Private, 202051, Masters, 14, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
30, Private, 54334, 9th, 5, Never-married, Sales, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Federal-gov, 410867, Doctorate, 16, Never-married, Prof-specialty, Not-in-family, White, Female, 0, 0, 50, United-States, >50K
|
||||
57, Private, 249977, Assoc-voc, 11, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
37, Private, 286730, Some-college, 10, Divorced, Craft-repair, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 212563, Some-college, 10, Divorced, Machine-op-inspct, Unmarried, Black, Female, 0, 0, 25, United-States, <=50K
|
||||
30, Private, 117747, HS-grad, 9, Married-civ-spouse, Sales, Wife, Asian-Pac-Islander, Female, 0, 1573, 35, ?, <=50K
|
||||
34, Local-gov, 226296, Bachelors, 13, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
29, Local-gov, 115585, Some-college, 10, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 50, United-States, <=50K
|
||||
48, Self-emp-not-inc, 191277, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 1902, 60, United-States, >50K
|
||||
37, Private, 202683, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, >50K
|
||||
48, Private, 171095, Assoc-acdm, 12, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, England, <=50K
|
||||
32, Federal-gov, 249409, HS-grad, 9, Never-married, Other-service, Own-child, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
76, Private, 124191, Masters, 14, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
|
|
|
@ -1,5 +0,0 @@
|
|||
customerId,customerName
|
||||
1,John
|
||||
2,Clerk
|
||||
3,Micheal
|
||||
4,Sample
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue