forked from opengaussexamples/examples
添加实现Spark的datasource端口的示例代码及验证，添加简单使用jdbc的spark代码示例
Signed-off-by: Cerdore <khn64@163.com>
This commit is contained in:
parent
e2c2076829
commit
8e88d94be1
|
|
@ -0,0 +1,10 @@
|
|||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Datasource local storage ignored files
|
||||
/../../../../:\gitRepo\SparkOpOG\.idea/dataSources/
|
||||
/dataSources.local.xml
|
||||
# Zeppelin ignored files
|
||||
/ZeppelinRemoteNotebooks/
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
<component name="ProjectCodeStyleConfiguration">
|
||||
<code_scheme name="Project" version="173">
|
||||
<ScalaCodeStyleSettings>
|
||||
<option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
|
||||
</ScalaCodeStyleSettings>
|
||||
</code_scheme>
|
||||
</component>
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
<component name="ProjectCodeStyleConfiguration">
|
||||
<state>
|
||||
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
|
||||
</state>
|
||||
</component>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CompilerConfiguration">
|
||||
<annotationProcessing>
|
||||
<profile name="Maven default annotation processors profile" enabled="true">
|
||||
<sourceOutputDir name="target/generated-sources/annotations" />
|
||||
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
|
||||
<outputRelativeToContentRoot value="true" />
|
||||
<module name="SparkOpOpenGauss" />
|
||||
<module name="SparkOpOG" />
|
||||
<module name="SparkOpOGForTest" />
|
||||
</profile>
|
||||
</annotationProcessing>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="opengauss@10.11.6.27" uuid="f5515d82-d7ba-4fc5-9425-f44775711cf3">
|
||||
<driver-ref>postgresql</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
|
||||
<jdbc-url>jdbc:postgresql://10.11.6.27:15432/postgres</jdbc-url>
|
||||
</data-source>
|
||||
</component>
|
||||
</project>
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1 @@
|
|||
#n:postgres
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#n:information_schema
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#n:pg_catalog
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#n:sparkuser
|
||||
!<md> [11790, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -0,0 +1 @@
|
|||
#n:school
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#n:information_schema
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#n:pg_catalog
|
||||
!<md> [null, 0, null, null, -2147483648, -2147483648]
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="GoogleJavaFormatSettings">
|
||||
<option name="enabled" value="false" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="RemoteRepositoriesConfiguration">
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Central Repository" />
|
||||
<option name="url" value="https://repo.maven.apache.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="Apache" />
|
||||
<option name="name" value="Apache" />
|
||||
<option name="url" value="https://repository.apache.org/snapshots/" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Maven Central repository" />
|
||||
<option name="url" value="https://repo1.maven.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="jboss.community" />
|
||||
<option name="name" value="JBoss Community repository" />
|
||||
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
|
||||
</remote-repository>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<component name="libraryTable">
|
||||
<library name="opengauss-jdbc-2.0.0">
|
||||
<CLASSES>
|
||||
<root url="jar://$PROJECT_DIR$/libs/opengauss-jdbc-2.0.0.jar!/" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</component>
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="MavenProjectsManager">
|
||||
<option name="originalFiles">
|
||||
<list>
|
||||
<option value="$PROJECT_DIR$/pom.xml" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Palette2">
|
||||
<group name="Swing">
|
||||
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
|
||||
</item>
|
||||
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
|
||||
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
|
||||
<initial-values>
|
||||
<property name="text" value="Button" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="RadioButton" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="CheckBox" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="Label" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
|
||||
<preferred-size width="-1" height="20" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
|
||||
</item>
|
||||
</group>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# SparkOpOpenGauss
|
||||
|
||||
这是将 openGuass 作为数据源的spark示例代码。
|
||||
|
||||
**环境和软件版本要求（前置要求）：**
|
||||
|
||||
1. Java 1.8
|
||||
2. Scala 2.12
|
||||
3. postgresql.jar 或 opengauss-jdbc--${version}.jar(自己打包或官方提供的jar包)
|
||||
|
||||
**说明：**
|
||||
|
||||
1. 请确保服务器上的数据库正常运行，且你的机器可正常连接数据库
|
||||
2. 以sparkuser用户身份执行./resources/school.sql文件
|
||||
3. 请修改代码中连接数据库的ip及端口，即修改 x.x.x.x:port 。主要代码内容在 ./src/main/scala/
|
||||
4. 可以在idea运行本示例。
|
||||
+ 可运行./src/test/scala/org/opengauss/spark/OpenGaussExample。
|
||||
+ 另一个直接使用Spark JDBC的例子是 ./src/main/scala/SQLDataSourceExample。
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="JAVA_MODULE" version="4" />
|
||||
Binary file not shown.
|
|
@ -0,0 +1,91 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>SparkOpOG</artifactId>
|
||||
<version>3.3.0-SNAPSHOT</version>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>8</source>
|
||||
<target>8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.scalatest</groupId>
|
||||
<artifactId>scalatest-maven-plugin</artifactId>
|
||||
<version>1.0</version>
|
||||
<configuration>
|
||||
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
|
||||
<junitxml>.</junitxml>
|
||||
<filereports>WDF TestSuite.txt</filereports>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>test</id>
|
||||
<goals>
|
||||
<goal>test</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
|
||||
<dependency>
|
||||
<groupId>org.scalatest</groupId>
|
||||
<artifactId>scalatest_2.12</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.12</artifactId>
|
||||
<version>3.3.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.12</artifactId>
|
||||
<version>3.3.0-SNAPSHOT</version>
|
||||
<!-- <scope>provided</scope>-->
|
||||
</dependency>
|
||||
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>org.postgresql</groupId>-->
|
||||
<!-- <artifactId>postgresql</artifactId>-->
|
||||
<!--<!– <version>42.2.23</version>–>-->
|
||||
<!-- <version>system</version>-->
|
||||
<!-- <systemPath>${project.basedir}/libs/opengauss-jdbc-2.0.0.jar</systemPath>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>org.postgresql</groupId>-->
|
||||
<!--<!– <artifactId></artifactId>–>-->
|
||||
<!--<!– <version>42.2.23</version>–>-->
|
||||
<!-- <version>system</version>-->
|
||||
<!-- <systemPath>${project.basedir}/libs/opengauss-jdbc-2.0.0.jar</systemPath>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
</dependencies>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>Apache</id>
|
||||
<url>https://repository.apache.org/snapshots/</url>
|
||||
<snapshots>
|
||||
<enabled>true</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,214 @@
|
|||
create database school;
|
||||
|
||||
\c school;
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- 创建表student
|
||||
CREATE TABLE student
|
||||
(
|
||||
std_id INT PRIMARY KEY,
|
||||
std_name VARCHAR(20) NOT NULL,
|
||||
std_sex VARCHAR(6),
|
||||
std_birth DATE,
|
||||
std_in DATE NOT NULL,
|
||||
std_address VARCHAR(100)
|
||||
);
|
||||
|
||||
-- 创建表teacher
|
||||
CREATE TABLE teacher
|
||||
(
|
||||
tec_id INT PRIMARY KEY,
|
||||
tec_name VARCHAR(20) NOT NULL,
|
||||
tec_job VARCHAR(15),
|
||||
tec_sex VARCHAR(6),
|
||||
tec_age INT,
|
||||
tec_in DATE NOT NULL
|
||||
);
|
||||
|
||||
-- 创建表class
|
||||
CREATE TABLE class
|
||||
(
|
||||
cla_id INT PRIMARY KEY,
|
||||
cla_name VARCHAR(20) NOT NULL,
|
||||
cla_teacher INT NOT NULL
|
||||
);
|
||||
-- 给表class添加外键约束
|
||||
ALTER TABLE class ADD CONSTRAINT fk_tec_id FOREIGN KEY (cla_teacher) REFERENCES teacher(tec_id) ON DELETE CASCADE;
|
||||
|
||||
-- 创建表school_department
|
||||
CREATE TABLE school_department
|
||||
(
|
||||
depart_id INT PRIMARY KEY,
|
||||
depart_name VARCHAR(30) NOT NULL,
|
||||
depart_teacher INT NOT NULL
|
||||
);
|
||||
-- 给表school_department添加外键约束
|
||||
ALTER TABLE school_department ADD CONSTRAINT fk_depart_tec_id FOREIGN KEY (depart_teacher) REFERENCES teacher(tec_id) ON DELETE CASCADE;
|
||||
|
||||
-- 创建表course
|
||||
CREATE TABLE course
|
||||
(
|
||||
cor_id INT PRIMARY KEY,
|
||||
cor_name VARCHAR(30) NOT NULL,
|
||||
cor_type VARCHAR(20),
|
||||
credit DOUBLE PRECISION
|
||||
);
|
||||
|
||||
-- 插入数据
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (1,'张一','男','1993-01-01','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (2,'张二','男','1993-01-02','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (3,'张三','男','1993-01-03','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (4,'张四','男','1993-01-04','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (5,'张五','男','1993-01-05','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (6,'张六','男','1993-01-06','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (7,'张七','男','1993-01-07','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (8,'张八','男','1993-01-08','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (9,'张九','男','1993-01-09','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (10,'李一','男','1993-01-10','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (11,'李二','男','1993-01-11','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (12,'李三','男','1993-01-12','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (13,'李四','男','1993-01-13','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (14,'李五','男','1993-01-14','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (15,'李六','男','1993-01-15','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (16,'李七','男','1993-01-16','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (17,'李八','男','1993-01-17','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (18,'李九','男','1993-01-18','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (19,'王一','男','1993-01-19','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (20,'王二','男','1993-01-20','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (21,'王三','男','1993-01-21','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (22,'王四','男','1993-01-22','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (23,'王五','男','1993-01-23','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (24,'王六','男','1993-01-24','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (25,'王七','男','1993-01-25','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (26,'王八','男','1993-01-26','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (27,'王九','男','1993-01-27','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (28,'钱一','男','1993-01-28','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (29,'钱二','男','1993-01-29','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (30,'钱三','男','1993-01-30','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (31,'钱四','男','1993-02-01','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (32,'钱五','男','1993-02-02','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (33,'钱六','男','1993-02-03','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (34,'钱七','男','1993-02-04','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (35,'钱八','男','1993-02-05','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (36,'钱九','男','1993-02-06','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (37,'吴一','男','1993-02-07','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (38,'吴二','男','1993-02-08','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (39,'吴三','男','1993-02-09','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (40,'吴四','男','1993-02-10','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (41,'吴五','男','1993-02-11','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (42,'吴六','男','1993-02-12','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (43,'吴七','男','1993-02-13','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (44,'吴八','男','1993-02-14','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (45,'吴九','男','1993-02-15','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (46,'柳一','男','1993-02-16','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (47,'柳二','男','1993-02-17','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (48,'柳三','男','1993-02-18','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (49,'柳四','男','1993-02-19','2011-09-01','江苏省南京市雨花台区');
|
||||
INSERT INTO student(std_id,std_name,std_sex,std_birth,std_in,std_address) VALUES (50,'柳五','男','1993-02-20','2011-09-01','江苏省南京市雨花台区');
|
||||
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (1,'张一','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (2,'张二','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (3,'张三','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (4,'张四','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (5,'张五','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (6,'张六','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (7,'张七','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (8,'张八','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (9,'张九','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (10,'李一','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (11,'李二','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (12,'李三','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (13,'李四','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (14,'李五','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (15,'李六','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (16,'李七','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (17,'李八','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (18,'李九','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (19,'王一','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (20,'王二','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (21,'王三','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (22,'王四','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (23,'王五','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (24,'王六','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (25,'王七','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (26,'王八','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (27,'王九','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (28,'钱一','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (29,'钱二','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (30,'钱三','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (31,'钱四','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (32,'钱五','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (33,'钱六','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (34,'钱七','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (35,'钱八','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (36,'钱九','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (37,'吴一','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (38,'吴二','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (39,'吴三','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (40,'吴四','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (41,'吴五','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (42,'吴六','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (43,'吴七','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (44,'吴八','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (45,'吴九','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (46,'柳一','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (47,'柳二','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (48,'柳三','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (49,'柳四','讲师','男',35,'2009-07-01');
|
||||
INSERT INTO teacher(tec_id,tec_name,tec_job,tec_sex,tec_age,tec_in) VALUES (50,'柳五','讲师','男',35,'2009-07-01');
|
||||
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (1,'计算机',1);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (2,'自动化',3);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (3,'飞行器设计',5);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (4,'大学物理',7);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (5,'高等数学',9);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (6,'大学化学',12);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (7,'表演',14);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (8,'服装设计',16);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (9,'工业设计',18);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (10,'金融学',21);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (11,'医学',23);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (12,'土木工程',25);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (13,'机械',27);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (14,'建筑学',29);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (15,'经济学',32);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (16,'财务管理',34);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (17,'人力资源',36);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (18,'力学',38);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (19,'人工智能',41);
|
||||
INSERT INTO class(cla_id,cla_name,cla_teacher) VALUES (20,'会计',45);
|
||||
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (1,'计算机学院',2);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (2,'自动化学院',4);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (3,'航空宇航学院',6);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (4,'艺术学院',8);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (5,'理学院',11);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (6,'人工智能学院',13);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (7,'工学院',15);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (8,'管理学院',17);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (9,'农学院',22);
|
||||
INSERT INTO school_department(depart_id,depart_name,depart_teacher) VALUES (10,'医学院',28);
|
||||
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (1,'数据库系统概论','必修',3);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (2,'艺术设计概论','选修',1);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (3,'力学制图','必修',4);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (4,'飞行器设计历史','选修',1);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (5,'马克思主义','必修',2);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (6,'大学历史','必修',2);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (7,'人力资源管理理论','必修',2.5);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (8,'线性代数','必修',4);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (9,'JAVA程序设计','必修',3);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (10,'操作系统','必修',4);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (11,'计算机组成原理','必修',3);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (12,'自动化设计理论','必修',2);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (13,'情绪表演','必修',2.5);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (14,'茶学历史','选修',1);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (15,'艺术论','必修',1.5);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (16,'机器学习','必修',3);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (17,'数据挖掘','选修',2);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (18,'图像识别','必修',3);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (19,'解剖学','必修',4);
|
||||
INSERT INTO course(cor_id,cor_name,cor_type,credit) VALUES (20,'3D max','选修',2);
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
age,workclass,fnlwgt,education,education_num,martial_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,salary
|
||||
39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White, Male, 2174, 0, 40, United-States, <=50K
|
||||
50, Self-emp-not-inc, 83311, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 13, United-States, <=50K
|
||||
38, Private, 215646, HS-grad, 9, Divorced, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 234721, 11th, 7, Married-civ-spouse, Handlers-cleaners, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 338409, Bachelors, 13, Married-civ-spouse, Prof-specialty, Wife, Black, Female, 0, 0, 40, Cuba, <=50K
|
||||
37, Private, 284582, Masters, 14, Married-civ-spouse, Exec-managerial, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
49, Private, 160187, 9th, 5, Married-spouse-absent, Other-service, Not-in-family, Black, Female, 0, 0, 16, Jamaica, <=50K
|
||||
52, Self-emp-not-inc, 209642, HS-grad, 9, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
31, Private, 45781, Masters, 14, Never-married, Prof-specialty, Not-in-family, White, Female, 14084, 0, 50, United-States, >50K
|
||||
42, Private, 159449, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 5178, 0, 40, United-States, >50K
|
||||
37, Private, 280464, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, Black, Male, 0, 0, 80, United-States, >50K
|
||||
30, State-gov, 141297, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Asian-Pac-Islander, Male, 0, 0, 40, India, >50K
|
||||
23, Private, 122272, Bachelors, 13, Never-married, Adm-clerical, Own-child, White, Female, 0, 0, 30, United-States, <=50K
|
||||
32, Private, 205019, Assoc-acdm, 12, Never-married, Sales, Not-in-family, Black, Male, 0, 0, 50, United-States, <=50K
|
||||
40, Private, 121772, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male, 0, 0, 40, ?, >50K
|
||||
34, Private, 245487, 7th-8th, 4, Married-civ-spouse, Transport-moving, Husband, Amer-Indian-Eskimo, Male, 0, 0, 45, Mexico, <=50K
|
||||
25, Self-emp-not-inc, 176756, HS-grad, 9, Never-married, Farming-fishing, Own-child, White, Male, 0, 0, 35, United-States, <=50K
|
||||
32, Private, 186824, HS-grad, 9, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, United-States, <=50K
|
||||
38, Private, 28887, 11th, 7, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
43, Self-emp-not-inc, 292175, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 45, United-States, >50K
|
||||
40, Private, 193524, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 60, United-States, >50K
|
||||
54, Private, 302146, HS-grad, 9, Separated, Other-service, Unmarried, Black, Female, 0, 0, 20, United-States, <=50K
|
||||
35, Federal-gov, 76845, 9th, 5, Married-civ-spouse, Farming-fishing, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Private, 117037, 11th, 7, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 2042, 40, United-States, <=50K
|
||||
59, Private, 109015, HS-grad, 9, Divorced, Tech-support, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
56, Local-gov, 216851, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
19, Private, 168294, HS-grad, 9, Never-married, Craft-repair, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
54, ?, 180211, Some-college, 10, Married-civ-spouse, ?, Husband, Asian-Pac-Islander, Male, 0, 0, 60, South, >50K
|
||||
39, Private, 367260, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 80, United-States, <=50K
|
||||
49, Private, 193366, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
23, Local-gov, 190709, Assoc-acdm, 12, Never-married, Protective-serv, Not-in-family, White, Male, 0, 0, 52, United-States, <=50K
|
||||
20, Private, 266015, Some-college, 10, Never-married, Sales, Own-child, Black, Male, 0, 0, 44, United-States, <=50K
|
||||
45, Private, 386940, Bachelors, 13, Divorced, Exec-managerial, Own-child, White, Male, 0, 1408, 40, United-States, <=50K
|
||||
30, Federal-gov, 59951, Some-college, 10, Married-civ-spouse, Adm-clerical, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
22, State-gov, 311512, Some-college, 10, Married-civ-spouse, Other-service, Husband, Black, Male, 0, 0, 15, United-States, <=50K
|
||||
48, Private, 242406, 11th, 7, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
21, Private, 197200, Some-college, 10, Never-married, Machine-op-inspct, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 544091, HS-grad, 9, Married-AF-spouse, Adm-clerical, Wife, White, Female, 0, 0, 25, United-States, <=50K
|
||||
31, Private, 84154, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 38, ?, >50K
|
||||
48, Self-emp-not-inc, 265477, Assoc-acdm, 12, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
31, Private, 507875, 9th, 5, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 43, United-States, <=50K
|
||||
53, Self-emp-not-inc, 88506, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
24, Private, 172987, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
49, Private, 94638, HS-grad, 9, Separated, Adm-clerical, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
25, Private, 289980, HS-grad, 9, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 35, United-States, <=50K
|
||||
57, Federal-gov, 337895, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Black, Male, 0, 0, 40, United-States, >50K
|
||||
53, Private, 144361, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 38, United-States, <=50K
|
||||
44, Private, 128354, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
41, State-gov, 101603, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
29, Private, 271466, Assoc-voc, 11, Never-married, Prof-specialty, Not-in-family, White, Male, 0, 0, 43, United-States, <=50K
|
||||
25, Private, 32275, Some-college, 10, Married-civ-spouse, Exec-managerial, Wife, Other, Female, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 226956, HS-grad, 9, Never-married, Other-service, Own-child, White, Female, 0, 0, 30, ?, <=50K
|
||||
47, Private, 51835, Prof-school, 15, Married-civ-spouse, Prof-specialty, Wife, White, Female, 0, 1902, 60, Honduras, >50K
|
||||
50, Federal-gov, 251585, Bachelors, 13, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 55, United-States, >50K
|
||||
47, Self-emp-inc, 109832, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 60, United-States, <=50K
|
||||
43, Private, 237993, Some-college, 10, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
46, Private, 216666, 5th-6th, 3, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
35, Private, 56352, Assoc-voc, 11, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
41, Private, 147372, HS-grad, 9, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
30, Private, 188146, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 5013, 0, 40, United-States, <=50K
|
||||
30, Private, 59496, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 2407, 0, 40, United-States, <=50K
|
||||
32, ?, 293936, 7th-8th, 4, Married-spouse-absent, ?, Not-in-family, White, Male, 0, 0, 40, ?, <=50K
|
||||
48, Private, 149640, HS-grad, 9, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
42, Private, 116632, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
29, Private, 105598, Some-college, 10, Divorced, Tech-support, Not-in-family, White, Male, 0, 0, 58, United-States, <=50K
|
||||
36, Private, 155537, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 183175, Some-college, 10, Divorced, Adm-clerical, Not-in-family, White, Female, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 169846, HS-grad, 9, Married-civ-spouse, Adm-clerical, Wife, White, Female, 0, 0, 40, United-States, >50K
|
||||
49, Self-emp-inc, 191681, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 50, United-States, >50K
|
||||
25, ?, 200681, Some-college, 10, Never-married, ?, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 101509, Some-college, 10, Never-married, Prof-specialty, Own-child, White, Male, 0, 0, 32, United-States, <=50K
|
||||
31, Private, 309974, Bachelors, 13, Separated, Sales, Own-child, Black, Female, 0, 0, 40, United-States, <=50K
|
||||
29, Self-emp-not-inc, 162298, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 70, United-States, >50K
|
||||
23, Private, 211678, Some-college, 10, Never-married, Machine-op-inspct, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
79, Private, 124744, Some-college, 10, Married-civ-spouse, Prof-specialty, Other-relative, White, Male, 0, 0, 20, United-States, <=50K
|
||||
27, Private, 213921, HS-grad, 9, Never-married, Other-service, Own-child, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
40, Private, 32214, Assoc-acdm, 12, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
67, ?, 212759, 10th, 6, Married-civ-spouse, ?, Husband, White, Male, 0, 0, 2, United-States, <=50K
|
||||
18, Private, 309634, 11th, 7, Never-married, Other-service, Own-child, White, Female, 0, 0, 22, United-States, <=50K
|
||||
31, Local-gov, 125927, 7th-8th, 4, Married-civ-spouse, Farming-fishing, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 446839, HS-grad, 9, Never-married, Sales, Not-in-family, White, Male, 0, 0, 30, United-States, <=50K
|
||||
52, Private, 276515, Bachelors, 13, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Cuba, <=50K
|
||||
46, Private, 51618, HS-grad, 9, Married-civ-spouse, Other-service, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
59, Private, 159937, HS-grad, 9, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
44, Private, 343591, HS-grad, 9, Divorced, Craft-repair, Not-in-family, White, Female, 14344, 0, 40, United-States, >50K
|
||||
53, Private, 346253, HS-grad, 9, Divorced, Sales, Own-child, White, Female, 0, 0, 35, United-States, <=50K
|
||||
49, Local-gov, 268234, HS-grad, 9, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
33, Private, 202051, Masters, 14, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
30, Private, 54334, 9th, 5, Never-married, Sales, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Federal-gov, 410867, Doctorate, 16, Never-married, Prof-specialty, Not-in-family, White, Female, 0, 0, 50, United-States, >50K
|
||||
57, Private, 249977, Assoc-voc, 11, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
37, Private, 286730, Some-college, 10, Divorced, Craft-repair, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 212563, Some-college, 10, Divorced, Machine-op-inspct, Unmarried, Black, Female, 0, 0, 25, United-States, <=50K
|
||||
30, Private, 117747, HS-grad, 9, Married-civ-spouse, Sales, Wife, Asian-Pac-Islander, Female, 0, 1573, 35, ?, <=50K
|
||||
34, Local-gov, 226296, Bachelors, 13, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
29, Local-gov, 115585, Some-college, 10, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 50, United-States, <=50K
|
||||
48, Self-emp-not-inc, 191277, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 1902, 60, United-States, >50K
|
||||
37, Private, 202683, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, >50K
|
||||
48, Private, 171095, Assoc-acdm, 12, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, England, <=50K
|
||||
32, Federal-gov, 249409, HS-grad, 9, Never-married, Other-service, Own-child, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
76, Private, 124191, Masters, 14, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
customerId,customerName
|
||||
1,John
|
||||
2,Clerk
|
||||
3,Micheal
|
||||
4,Sample
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
transactionId,customerId,itemId,amountPaid
|
||||
111,1,1,100.0
|
||||
112,2,2,505.0
|
||||
113,3,3,510.0
|
||||
114,4,4,600.0
|
||||
115,1,2,500.0
|
||||
116,1,2,500.0
|
||||
117,1,2,500.0
|
||||
118,1,2,500.0
|
||||
119,2,3,500.0
|
||||
120,1,2,500.0
|
||||
121,1,4,500.0
|
||||
122,1,2,500.0
|
||||
123,1,4,500.0
|
||||
124,1,2,500.0
|
||||
|
|
|
@ -0,0 +1,400 @@
|
|||
package org.apache.spark.examples.sql
|
||||
|
||||
import java.util.Properties
|
||||
|
||||
import org.apache.spark.sql.SparkSession
|
||||
|
||||
object SQLDataSourceExample {
|
||||
|
||||
case class Person(name: String, age: Long)
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
val spark = SparkSession
|
||||
.builder()
|
||||
.master("local")
|
||||
.appName("Spark SQL data sources example")
|
||||
.config("spark.some.config.option", "some-value")
|
||||
.getOrCreate()
|
||||
|
||||
// runBasicDataSourceExample(spark)
|
||||
// runGenericFileSourceOptionsExample(spark)
|
||||
// runBasicParquetExample(spark)
|
||||
// runParquetSchemaMergingExample(spark)
|
||||
// runJsonDatasetExample(spark)
|
||||
// runCsvDatasetExample(spark)
|
||||
// runTextDatasetExample(spark)
|
||||
runJdbcDatasetExample(spark)
|
||||
|
||||
spark.stop()
|
||||
}
|
||||
|
||||
private def runGenericFileSourceOptionsExample(spark: SparkSession): Unit = {
|
||||
// $example on:ignore_corrupt_files$
|
||||
// enable ignore corrupt files
|
||||
spark.sql("set spark.sql.files.ignoreCorruptFiles=true")
|
||||
// dir1/file3.json is corrupt from parquet's view
|
||||
val testCorruptDF = spark.read.parquet(
|
||||
"examples/src/main/resources/dir1/",
|
||||
"examples/src/main/resources/dir1/dir2/")
|
||||
testCorruptDF.show()
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// |file2.parquet|
|
||||
// +-------------+
|
||||
// $example off:ignore_corrupt_files$
|
||||
// $example on:recursive_file_lookup$
|
||||
val recursiveLoadedDF = spark.read.format("parquet")
|
||||
.option("recursiveFileLookup", "true")
|
||||
.load("examples/src/main/resources/dir1")
|
||||
recursiveLoadedDF.show()
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// |file2.parquet|
|
||||
// +-------------+
|
||||
// $example off:recursive_file_lookup$
|
||||
spark.sql("set spark.sql.files.ignoreCorruptFiles=false")
|
||||
// $example on:load_with_path_glob_filter$
|
||||
val testGlobFilterDF = spark.read.format("parquet")
|
||||
.option("pathGlobFilter", "*.parquet") // json file should be filtered out
|
||||
.load("examples/src/main/resources/dir1")
|
||||
testGlobFilterDF.show()
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// +-------------+
|
||||
// $example off:load_with_path_glob_filter$
|
||||
// $example on:load_with_modified_time_filter$
|
||||
val beforeFilterDF = spark.read.format("parquet")
|
||||
// Files modified before 07/01/2020 at 05:30 are allowed
|
||||
.option("modifiedBefore", "2020-07-01T05:30:00")
|
||||
.load("examples/src/main/resources/dir1");
|
||||
beforeFilterDF.show();
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// |file1.parquet|
|
||||
// +-------------+
|
||||
val afterFilterDF = spark.read.format("parquet")
|
||||
// Files modified after 06/01/2020 at 05:30 are allowed
|
||||
.option("modifiedAfter", "2020-06-01T05:30:00")
|
||||
.load("examples/src/main/resources/dir1");
|
||||
afterFilterDF.show();
|
||||
// +-------------+
|
||||
// | file|
|
||||
// +-------------+
|
||||
// +-------------+
|
||||
// $example off:load_with_modified_time_filter$
|
||||
}
|
||||
|
||||
private def runBasicDataSourceExample(spark: SparkSession): Unit = {
|
||||
// $example on:generic_load_save_functions$
|
||||
val usersDF = spark.read.load("examples/src/main/resources/users.parquet")
|
||||
usersDF.select("name", "favorite_color").write.save("namesAndFavColors.parquet")
|
||||
// $example off:generic_load_save_functions$
|
||||
// $example on:manual_load_options$
|
||||
val peopleDF = spark.read.format("json").load("examples/src/main/resources/people.json")
|
||||
peopleDF.select("name", "age").write.format("parquet").save("namesAndAges.parquet")
|
||||
// $example off:manual_load_options$
|
||||
// $example on:manual_load_options_csv$
|
||||
val peopleDFCsv = spark.read.format("csv")
|
||||
.option("sep", ";")
|
||||
.option("inferSchema", "true")
|
||||
.option("header", "true")
|
||||
.load("examples/src/main/resources/people.csv")
|
||||
// $example off:manual_load_options_csv$
|
||||
// $example on:manual_save_options_orc$
|
||||
usersDF.write.format("orc")
|
||||
.option("orc.bloom.filter.columns", "favorite_color")
|
||||
.option("orc.dictionary.key.threshold", "1.0")
|
||||
.option("orc.column.encoding.direct", "name")
|
||||
.save("users_with_options.orc")
|
||||
// $example off:manual_save_options_orc$
|
||||
// $example on:manual_save_options_parquet$
|
||||
usersDF.write.format("parquet")
|
||||
.option("parquet.bloom.filter.enabled#favorite_color", "true")
|
||||
.option("parquet.bloom.filter.expected.ndv#favorite_color", "1000000")
|
||||
.option("parquet.enable.dictionary", "true")
|
||||
.option("parquet.page.write-checksum.enabled", "false")
|
||||
.save("users_with_options.parquet")
|
||||
// $example off:manual_save_options_parquet$
|
||||
|
||||
// $example on:direct_sql$
|
||||
val sqlDF = spark.sql("SELECT * FROM parquet.`examples/src/main/resources/users.parquet`")
|
||||
// $example off:direct_sql$
|
||||
// $example on:write_sorting_and_bucketing$
|
||||
peopleDF.write.bucketBy(42, "name").sortBy("age").saveAsTable("people_bucketed")
|
||||
// $example off:write_sorting_and_bucketing$
|
||||
// $example on:write_partitioning$
|
||||
usersDF.write.partitionBy("favorite_color").format("parquet").save("namesPartByColor.parquet")
|
||||
// $example off:write_partitioning$
|
||||
// $example on:write_partition_and_bucket$
|
||||
usersDF
|
||||
.write
|
||||
.partitionBy("favorite_color")
|
||||
.bucketBy(42, "name")
|
||||
.saveAsTable("users_partitioned_bucketed")
|
||||
// $example off:write_partition_and_bucket$
|
||||
|
||||
spark.sql("DROP TABLE IF EXISTS people_bucketed")
|
||||
spark.sql("DROP TABLE IF EXISTS users_partitioned_bucketed")
|
||||
}
|
||||
|
||||
private def runBasicParquetExample(spark: SparkSession): Unit = {
|
||||
// $example on:basic_parquet_example$
|
||||
// Encoders for most common types are automatically provided by importing spark.implicits._
|
||||
import spark.implicits._
|
||||
|
||||
val peopleDF = spark.read.json("examples/src/main/resources/people.json")
|
||||
|
||||
// DataFrames can be saved as Parquet files, maintaining the schema information
|
||||
peopleDF.write.parquet("people.parquet")
|
||||
|
||||
// Read in the parquet file created above
|
||||
// Parquet files are self-describing so the schema is preserved
|
||||
// The result of loading a Parquet file is also a DataFrame
|
||||
val parquetFileDF = spark.read.parquet("people.parquet")
|
||||
|
||||
// Parquet files can also be used to create a temporary view and then used in SQL statements
|
||||
parquetFileDF.createOrReplaceTempView("parquetFile")
|
||||
val namesDF = spark.sql("SELECT name FROM parquetFile WHERE age BETWEEN 13 AND 19")
|
||||
namesDF.map(attributes => "Name: " + attributes(0)).show()
|
||||
// +------------+
|
||||
// | value|
|
||||
// +------------+
|
||||
// |Name: Justin|
|
||||
// +------------+
|
||||
// $example off:basic_parquet_example$
|
||||
}
|
||||
|
||||
private def runParquetSchemaMergingExample(spark: SparkSession): Unit = {
|
||||
// $example on:schema_merging$
|
||||
// This is used to implicitly convert an RDD to a DataFrame.
|
||||
import spark.implicits._
|
||||
|
||||
// Create a simple DataFrame, store into a partition directory
|
||||
val squaresDF = spark.sparkContext.makeRDD(1 to 5).map(i => (i, i * i)).toDF("value", "square")
|
||||
squaresDF.write.parquet("data/test_table/key=1")
|
||||
|
||||
// Create another DataFrame in a new partition directory,
|
||||
// adding a new column and dropping an existing column
|
||||
val cubesDF = spark.sparkContext.makeRDD(6 to 10).map(i => (i, i * i * i)).toDF("value", "cube")
|
||||
cubesDF.write.parquet("data/test_table/key=2")
|
||||
|
||||
// Read the partitioned table
|
||||
val mergedDF = spark.read.option("mergeSchema", "true").parquet("data/test_table")
|
||||
mergedDF.printSchema()
|
||||
|
||||
// The final schema consists of all 3 columns in the Parquet files together
|
||||
// with the partitioning column appeared in the partition directory paths
|
||||
// root
|
||||
// |-- value: int (nullable = true)
|
||||
// |-- square: int (nullable = true)
|
||||
// |-- cube: int (nullable = true)
|
||||
// |-- key: int (nullable = true)
|
||||
// $example off:schema_merging$
|
||||
}
|
||||
|
||||
private def runJsonDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:json_dataset$
|
||||
// Primitive types (Int, String, etc) and Product types (case classes) encoders are
|
||||
// supported by importing this when creating a Dataset.
|
||||
import spark.implicits._
|
||||
|
||||
// A JSON dataset is pointed to by path.
|
||||
// The path can be either a single text file or a directory storing text files
|
||||
val path = "examples/src/main/resources/people.json"
|
||||
val peopleDF = spark.read.json(path)
|
||||
|
||||
// The inferred schema can be visualized using the printSchema() method
|
||||
peopleDF.printSchema()
|
||||
// root
|
||||
// |-- age: long (nullable = true)
|
||||
// |-- name: string (nullable = true)
|
||||
|
||||
// Creates a temporary view using the DataFrame
|
||||
peopleDF.createOrReplaceTempView("people")
|
||||
|
||||
// SQL statements can be run by using the sql methods provided by spark
|
||||
val teenagerNamesDF = spark.sql("SELECT name FROM people WHERE age BETWEEN 13 AND 19")
|
||||
teenagerNamesDF.show()
|
||||
// +------+
|
||||
// | name|
|
||||
// +------+
|
||||
// |Justin|
|
||||
// +------+
|
||||
|
||||
// Alternatively, a DataFrame can be created for a JSON dataset represented by
|
||||
// a Dataset[String] storing one JSON object per string
|
||||
val otherPeopleDataset = spark.createDataset(
|
||||
"""{"name":"Yin","address":{"city":"Columbus","state":"Ohio"}}""" :: Nil)
|
||||
val otherPeople = spark.read.json(otherPeopleDataset)
|
||||
otherPeople.show()
|
||||
// +---------------+----+
|
||||
// | address|name|
|
||||
// +---------------+----+
|
||||
// |[Columbus,Ohio]| Yin|
|
||||
// +---------------+----+
|
||||
// $example off:json_dataset$
|
||||
}
|
||||
|
||||
private def runCsvDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:csv_dataset$
|
||||
// A CSV dataset is pointed to by path.
|
||||
// The path can be either a single CSV file or a directory of CSV files
|
||||
val path = "examples/src/main/resources/people.csv"
|
||||
|
||||
val df = spark.read.csv(path)
|
||||
df.show()
|
||||
// +------------------+
|
||||
// | _c0|
|
||||
// +------------------+
|
||||
// | name;age;job|
|
||||
// |Jorge;30;Developer|
|
||||
// | Bob;32;Developer|
|
||||
// +------------------+
|
||||
|
||||
// Read a csv with delimiter, the default delimiter is ","
|
||||
val df2 = spark.read.option("delimiter", ";").csv(path)
|
||||
df2.show()
|
||||
// +-----+---+---------+
|
||||
// | _c0|_c1| _c2|
|
||||
// +-----+---+---------+
|
||||
// | name|age| job|
|
||||
// |Jorge| 30|Developer|
|
||||
// | Bob| 32|Developer|
|
||||
// +-----+---+---------+
|
||||
|
||||
// Read a csv with delimiter and a header
|
||||
val df3 = spark.read.option("delimiter", ";").option("header", "true").csv(path)
|
||||
df3.show()
|
||||
// +-----+---+---------+
|
||||
// | name|age| job|
|
||||
// +-----+---+---------+
|
||||
// |Jorge| 30|Developer|
|
||||
// | Bob| 32|Developer|
|
||||
// +-----+---+---------+
|
||||
|
||||
// You can also use options() to use multiple options
|
||||
val df4 = spark.read.options(Map("delimiter" -> ";", "header" -> "true")).csv(path)
|
||||
|
||||
// "output" is a folder which contains multiple csv files and a _SUCCESS file.
|
||||
df3.write.csv("output")
|
||||
|
||||
// Read all files in a folder, please make sure only CSV files should present in the folder.
|
||||
val folderPath = "examples/src/main/resources";
|
||||
val df5 = spark.read.csv(folderPath);
|
||||
df5.show();
|
||||
// Wrong schema because non-CSV files are read
|
||||
// +-----------+
|
||||
// | _c0|
|
||||
// +-----------+
|
||||
// |238val_238|
|
||||
// | 86val_86|
|
||||
// |311val_311|
|
||||
// | 27val_27|
|
||||
// |165val_165|
|
||||
// +-----------+
|
||||
|
||||
// $example off:csv_dataset$
|
||||
}
|
||||
|
||||
private def runTextDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:text_dataset$
|
||||
// A text dataset is pointed to by path.
|
||||
// The path can be either a single text file or a directory of text files
|
||||
val path = "examples/src/main/resources/people.txt"
|
||||
|
||||
val df1 = spark.read.text(path)
|
||||
df1.show()
|
||||
// +-----------+
|
||||
// | value|
|
||||
// +-----------+
|
||||
// |Michael, 29|
|
||||
// | Andy, 30|
|
||||
// | Justin, 19|
|
||||
// +-----------+
|
||||
|
||||
// You can use 'lineSep' option to define the line separator.
|
||||
// The line separator handles all `\r`, `\r\n` and `\n` by default.
|
||||
val df2 = spark.read.option("lineSep", ",").text(path)
|
||||
df2.show()
|
||||
// +-----------+
|
||||
// | value|
|
||||
// +-----------+
|
||||
// | Michael|
|
||||
// | 29\nAndy|
|
||||
// | 30\nJustin|
|
||||
// | 19\n|
|
||||
// +-----------+
|
||||
|
||||
// You can also use 'wholetext' option to read each input file as a single row.
|
||||
val df3 = spark.read.option("wholetext", true).text(path)
|
||||
df3.show()
|
||||
// +--------------------+
|
||||
// | value|
|
||||
// +--------------------+
|
||||
// |Michael, 29\nAndy...|
|
||||
// +--------------------+
|
||||
|
||||
// "output" is a folder which contains multiple text files and a _SUCCESS file.
|
||||
df1.write.text("output")
|
||||
|
||||
// You can specify the compression format using the 'compression' option.
|
||||
df1.write.option("compression", "gzip").text("output_compressed")
|
||||
|
||||
// $example off:text_dataset$
|
||||
}
|
||||
|
||||
private def runJdbcDatasetExample(spark: SparkSession): Unit = {
|
||||
// $example on:jdbc_dataset$
|
||||
// Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
|
||||
// Loading data from a JDBC source
|
||||
val jdbcDF = spark.read
|
||||
.format("jdbc")
|
||||
.option("url", "jdbc:postgresql://x.x.x.x:port/school")
|
||||
.option("dbtable", "class")
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "Enmo@123")
|
||||
.load()
|
||||
//.show()
|
||||
|
||||
val connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "sparkuser")
|
||||
connectionProperties.put("password", "Enmo@123")
|
||||
val jdbcDF2 = spark.read
|
||||
.option("customSchema","cla_id INT")
|
||||
.jdbc("jdbc:postgresql://x.x.x.x:port/school", "class", connectionProperties).show()
|
||||
// Specifying the custom data types of the read schema
|
||||
|
||||
|
||||
connectionProperties.put("customSchema", "cla_id INT")//, cla_name STRING")
|
||||
val jdbcDF3 = spark.read
|
||||
.jdbc("jdbc:postgresql://x.x.x.x:port/school", "class", connectionProperties)
|
||||
|
||||
// // Saving data to a JDBC source. Create table "customtable1", and write data
|
||||
// jdbcDF.write
|
||||
// .format("jdbc")
|
||||
// .option("url", "jdbc:postgresql://x.x.x.x:port/school")
|
||||
// .option("dbtable", "customtable1")
|
||||
// .option("user", "sparkuser")
|
||||
// .option("password", "Enmo@123")
|
||||
// .save()
|
||||
//
|
||||
//
|
||||
// jdbcDF2.write
|
||||
// .jdbc("jdbc:postgresql://x.x.x.x:port/school", "customtable2", connectionProperties)
|
||||
|
||||
// Specifying create table column data types on write
|
||||
jdbcDF3.show()
|
||||
|
||||
// jdbcDF3.write
|
||||
// .option("createTableColumnTypes", "cla_id INT, cla_name VARCHAR(20)")
|
||||
// .jdbc("jdbc:postgresql://x.x.x.x:port/school", "customtable3", connectionProperties)
|
||||
// // $example off:jdbc_dataset$
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package org.opengauss.spark.sources.datasourcev2.multi
|
||||
|
||||
import java.util
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider}
|
||||
import org.apache.spark.sql.connector.expressions.Transform
|
||||
import org.apache.spark.sql.connector.read._
|
||||
import org.apache.spark.sql.types.{StringType, StructField, StructType}
|
||||
import org.apache.spark.sql.util.CaseInsensitiveStringMap
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
/*
|
||||
* Default source should some kind of relation provider
|
||||
*/
|
||||
class DefaultSource extends TableProvider{
|
||||
|
||||
override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType =
|
||||
getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema()
|
||||
|
||||
override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table =
|
||||
new SimpleBatchTable()
|
||||
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Defines Read Support and Initial Schema
|
||||
*/
|
||||
|
||||
class SimpleBatchTable extends Table with SupportsRead {
|
||||
override def name(): String = this.getClass.toString
|
||||
|
||||
override def schema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def capabilities(): util.Set[TableCapability] = Set(TableCapability.BATCH_READ).asJava
|
||||
|
||||
override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder()
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Scan object with no mixins
|
||||
*/
|
||||
class SimpleScanBuilder extends ScanBuilder {
|
||||
override def build(): Scan = new SimpleScan
|
||||
}
|
||||
|
||||
/*
|
||||
Batch Reading Support
|
||||
|
||||
The schema is repeated here as it can change after column pruning etc
|
||||
*/
|
||||
|
||||
class SimpleScan extends Scan with Batch{
|
||||
override def readSchema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def toBatch: Batch = this
|
||||
|
||||
override def planInputPartitions(): Array[InputPartition] = {
|
||||
Array(new SimplePartition(0,4),
|
||||
new SimplePartition(5,9))
|
||||
}
|
||||
override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory()
|
||||
}
|
||||
|
||||
// simple class to organise the partition
|
||||
class SimplePartition(val start:Int, val end:Int) extends InputPartition
|
||||
|
||||
// reader factory
|
||||
class SimplePartitionReaderFactory extends PartitionReaderFactory {
|
||||
override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new
|
||||
SimplePartitionReader(partition.asInstanceOf[SimplePartition])
|
||||
}
|
||||
|
||||
|
||||
// parathion reader
|
||||
class SimplePartitionReader(inputPartition: SimplePartition) extends PartitionReader[InternalRow] {
|
||||
|
||||
val values = Array("1", "2", "3", "4", "5","6","7","8","9","10")
|
||||
|
||||
var index = inputPartition.start
|
||||
|
||||
def next = index <= inputPartition.end
|
||||
|
||||
def get = {
|
||||
val stringValue = values(index)
|
||||
val stringUtf = UTF8String.fromString(stringValue)
|
||||
val row = InternalRow(stringUtf)
|
||||
index = index + 1
|
||||
row
|
||||
}
|
||||
|
||||
def close() = Unit
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
package org.opengauss.spark.sources.datasourcev2.simple
|
||||
|
||||
import java.util
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider}
|
||||
import org.apache.spark.sql.connector.expressions.Transform
|
||||
import org.apache.spark.sql.connector.read._
|
||||
import org.apache.spark.sql.types.{StringType, StructField, StructType}
|
||||
import org.apache.spark.sql.util.CaseInsensitiveStringMap
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
/*
|
||||
* Default source should some kind of relation provider
|
||||
*/
|
||||
class DefaultSource extends TableProvider{
|
||||
|
||||
override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType =
|
||||
getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema()
|
||||
|
||||
override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table =
|
||||
new SimpleBatchTable()
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Defines Read Support and Initial Schema
|
||||
*/
|
||||
|
||||
class SimpleBatchTable extends Table with SupportsRead {
|
||||
override def name(): String = this.getClass.toString
|
||||
|
||||
override def schema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def capabilities(): util.Set[TableCapability] = Set(TableCapability.BATCH_READ).asJava
|
||||
|
||||
override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder()
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*
|
||||
Scan object with no mixins
|
||||
*/
|
||||
class SimpleScanBuilder extends ScanBuilder {
|
||||
override def build(): Scan = new SimpleScan
|
||||
}
|
||||
|
||||
/*
|
||||
Batch Reading Support
|
||||
|
||||
The schema is repeated here as it can change after column pruning etc
|
||||
*/
|
||||
|
||||
class SimpleScan extends Scan with Batch{
|
||||
override def readSchema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def toBatch: Batch = this
|
||||
|
||||
override def planInputPartitions(): Array[InputPartition] = {
|
||||
Array(new SimplePartition())
|
||||
}
|
||||
override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory()
|
||||
}
|
||||
|
||||
// simple class to organise the partition
|
||||
class SimplePartition extends InputPartition
|
||||
|
||||
// reader factory
|
||||
class SimplePartitionReaderFactory extends PartitionReaderFactory {
|
||||
override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new SimplePartitionReader
|
||||
}
|
||||
|
||||
|
||||
// parathion reader
|
||||
class SimplePartitionReader extends PartitionReader[InternalRow] {
|
||||
|
||||
val values = Array("1", "2", "3", "4", "5")
|
||||
|
||||
var index = 0
|
||||
|
||||
def next = index < values.length
|
||||
|
||||
def get = {
|
||||
val stringValue = values(index)
|
||||
val stringUtf = UTF8String.fromString(stringValue)
|
||||
val row = InternalRow(stringUtf)
|
||||
index = index + 1
|
||||
row
|
||||
}
|
||||
|
||||
def close() = Unit
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package org.opengauss.spark.sources.datasourcev2.streamandbatch
|
||||
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.streaming.OutputMode
|
||||
|
||||
object DataSourceV2StreamAndBatchExample {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sparkSession = SparkSession.builder.
|
||||
master("local[2]")
|
||||
.appName("streaming example")
|
||||
.getOrCreate()
|
||||
|
||||
|
||||
val dataSource = "cn.ecnu.spark.sources.datasourcev2.streamandbatch.simple"
|
||||
|
||||
val batchDf = sparkSession
|
||||
.read
|
||||
.format(dataSource)
|
||||
.load()
|
||||
|
||||
batchDf.show()
|
||||
|
||||
val streamingDf = sparkSession.
|
||||
readStream.
|
||||
format(dataSource)
|
||||
.load()
|
||||
|
||||
val query = streamingDf.writeStream
|
||||
.format("console")
|
||||
.queryName("simple_source")
|
||||
.outputMode(OutputMode.Append())
|
||||
|
||||
query.start().awaitTermination()
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
package org.opengauss.spark.sources.datasourcev2.streamandbatch
|
||||
|
||||
import java.util
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider}
|
||||
import org.apache.spark.sql.connector.expressions.Transform
|
||||
import org.apache.spark.sql.connector.read._
|
||||
import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset}
|
||||
import org.apache.spark.sql.types.{StringType, StructField, StructType}
|
||||
import org.apache.spark.sql.util.CaseInsensitiveStringMap
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
/*
|
||||
* Default source should some kind of relation provider
|
||||
*/
|
||||
class DefaultSource extends TableProvider{
|
||||
|
||||
override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType =
|
||||
getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema()
|
||||
|
||||
override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table =
|
||||
new SimpleStreamingTable()
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Defines Read Support and Initial Schema
|
||||
*/
|
||||
|
||||
class SimpleStreamingTable extends Table with SupportsRead {
|
||||
override def name(): String = this.getClass.toString
|
||||
|
||||
override def schema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def capabilities(): util.Set[TableCapability] = Set(TableCapability.MICRO_BATCH_READ,
|
||||
TableCapability.BATCH_READ).asJava
|
||||
|
||||
override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder()
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Scan object with no mixins
|
||||
*/
|
||||
class SimpleScanBuilder extends ScanBuilder {
|
||||
override def build(): Scan = new SimpleScan
|
||||
}
|
||||
|
||||
/*
|
||||
Batch Reading Support
|
||||
|
||||
The schema is repeated here as it can change after column pruning etc
|
||||
*/
|
||||
|
||||
class SimpleScan extends Scan{
|
||||
override def readSchema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def toMicroBatchStream(checkpointLocation: String): MicroBatchStream = new SimpleMicroBatchStream()
|
||||
|
||||
override def toBatch: Batch = new SimpleBatch
|
||||
}
|
||||
|
||||
class SimpleBatch extends Batch{
|
||||
override def planInputPartitions(): Array[InputPartition] = Array(new SimplePartition)
|
||||
|
||||
override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory
|
||||
}
|
||||
|
||||
class SimpleOffset(value:Int) extends Offset {
|
||||
override def json(): String = s"""{"value":"$value"}"""
|
||||
}
|
||||
|
||||
class SimpleMicroBatchStream extends MicroBatchStream {
|
||||
var latestOffsetValue = 0
|
||||
|
||||
override def latestOffset(): Offset = {
|
||||
latestOffsetValue += 10
|
||||
new SimpleOffset(latestOffsetValue)
|
||||
}
|
||||
|
||||
override def planInputPartitions(offset: Offset, offset1: Offset): Array[InputPartition] = Array(new SimplePartition)
|
||||
|
||||
override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory()
|
||||
|
||||
override def initialOffset(): Offset = new SimpleOffset(latestOffsetValue)
|
||||
|
||||
override def deserializeOffset(s: String): Offset = new SimpleOffset(latestOffsetValue)
|
||||
|
||||
override def commit(offset: Offset): Unit = {}
|
||||
|
||||
override def stop(): Unit = {}
|
||||
}
|
||||
|
||||
|
||||
// simple class to organise the partition
|
||||
class SimplePartition extends InputPartition
|
||||
|
||||
// reader factory
|
||||
class SimplePartitionReaderFactory extends PartitionReaderFactory {
|
||||
override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new SimplePartitionReader
|
||||
}
|
||||
|
||||
|
||||
// parathion reader
|
||||
class SimplePartitionReader extends PartitionReader[InternalRow] {
|
||||
|
||||
val values = Array("1", "2", "3", "4", "5")
|
||||
|
||||
var index = 0
|
||||
|
||||
def next = index < values.length
|
||||
|
||||
def get = {
|
||||
val stringValue = values(index)
|
||||
val stringUtf = UTF8String.fromString(stringValue)
|
||||
val row = InternalRow(stringUtf)
|
||||
index = index + 1
|
||||
row
|
||||
}
|
||||
|
||||
def close() = Unit
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package org.opengauss.spark.sources.datasourcev2.streaming
|
||||
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.streaming.OutputMode
|
||||
|
||||
object DataSourceV2StreamingExample {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sparkSession = SparkSession.builder.
|
||||
master("local[2]")
|
||||
.appName("streaming example")
|
||||
.getOrCreate()
|
||||
|
||||
val streamingDf = sparkSession.
|
||||
readStream.
|
||||
format("cn.ecnu.spark.sources.datasourcev2.streaming.simple")
|
||||
.load()
|
||||
|
||||
|
||||
|
||||
val query = streamingDf.writeStream
|
||||
.format("console")
|
||||
.queryName("simple_source")
|
||||
.outputMode(OutputMode.Append())
|
||||
|
||||
query.start().awaitTermination()
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
package org.opengauss.spark.sources.datasourcev2.streaming
|
||||
|
||||
import java.util
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapability, TableProvider}
|
||||
import org.apache.spark.sql.connector.expressions.Transform
|
||||
import org.apache.spark.sql.connector.read._
|
||||
import org.apache.spark.sql.connector.read.streaming.{MicroBatchStream, Offset}
|
||||
import org.apache.spark.sql.types.{StringType, StructField, StructType}
|
||||
import org.apache.spark.sql.util.CaseInsensitiveStringMap
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
/*
|
||||
* Default source should some kind of relation provider
|
||||
*/
|
||||
class DefaultSource extends TableProvider{
|
||||
|
||||
override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType =
|
||||
getTable(null,Array.empty[Transform],caseInsensitiveStringMap.asCaseSensitiveMap()).schema()
|
||||
|
||||
override def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table =
|
||||
new SimpleStreamingTable()
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Defines Read Support and Initial Schema
|
||||
*/
|
||||
|
||||
class SimpleStreamingTable extends Table with SupportsRead {
|
||||
override def name(): String = this.getClass.toString
|
||||
|
||||
override def schema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def capabilities(): util.Set[TableCapability] = Set(TableCapability.MICRO_BATCH_READ).asJava
|
||||
|
||||
override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new SimpleScanBuilder()
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Scan object with no mixins
|
||||
*/
|
||||
class SimpleScanBuilder extends ScanBuilder {
|
||||
override def build(): Scan = new SimpleScan
|
||||
}
|
||||
|
||||
/*
|
||||
Batch Reading Support
|
||||
|
||||
The schema is repeated here as it can change after column pruning etc
|
||||
*/
|
||||
|
||||
class SimpleScan extends Scan{
|
||||
override def readSchema(): StructType = StructType(Array(StructField("value", StringType)))
|
||||
|
||||
override def toMicroBatchStream(checkpointLocation: String): MicroBatchStream = new SimpleMicroBatchStream()
|
||||
}
|
||||
|
||||
class SimpleOffset(value:Int) extends Offset {
|
||||
override def json(): String = s"""{"value":"$value"}"""
|
||||
}
|
||||
|
||||
class SimpleMicroBatchStream extends MicroBatchStream {
|
||||
var latestOffsetValue = 0
|
||||
|
||||
override def latestOffset(): Offset = {
|
||||
latestOffsetValue += 10
|
||||
new SimpleOffset(latestOffsetValue)
|
||||
}
|
||||
|
||||
override def planInputPartitions(offset: Offset, offset1: Offset): Array[InputPartition] = Array(new SimplePartition)
|
||||
|
||||
override def createReaderFactory(): PartitionReaderFactory = new SimplePartitionReaderFactory()
|
||||
|
||||
override def initialOffset(): Offset = new SimpleOffset(latestOffsetValue)
|
||||
|
||||
override def deserializeOffset(s: String): Offset = new SimpleOffset(latestOffsetValue)
|
||||
|
||||
override def commit(offset: Offset): Unit = {}
|
||||
|
||||
override def stop(): Unit = {}
|
||||
}
|
||||
|
||||
|
||||
// simple class to organise the partition
|
||||
class SimplePartition extends InputPartition
|
||||
|
||||
// reader factory
|
||||
class SimplePartitionReaderFactory extends PartitionReaderFactory {
|
||||
override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new SimplePartitionReader
|
||||
}
|
||||
|
||||
|
||||
// parathion reader
|
||||
class SimplePartitionReader extends PartitionReader[InternalRow] {
|
||||
|
||||
val values = Array("1", "2", "3", "4", "5")
|
||||
|
||||
var index = 0
|
||||
|
||||
def next = index < values.length
|
||||
|
||||
def get = {
|
||||
val stringValue = values(index)
|
||||
val stringUtf = UTF8String.fromString(stringValue)
|
||||
val row = InternalRow(stringUtf)
|
||||
index = index + 1
|
||||
row
|
||||
}
|
||||
|
||||
def close() = Unit
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
package org.opengauss.spark.sources.opengauss
|
||||
|
||||
import java.sql.DriverManager
|
||||
import java.util
|
||||
|
||||
import org.apache.spark.sql.catalyst.InternalRow
|
||||
import org.apache.spark.sql.connector.catalog._
|
||||
import org.apache.spark.sql.connector.expressions.Transform
|
||||
import org.apache.spark.sql.connector.read._
|
||||
import org.apache.spark.sql.connector.write._
|
||||
import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, StructField, StructType}
|
||||
import org.apache.spark.sql.util.CaseInsensitiveStringMap
|
||||
import org.apache.spark.unsafe.types.UTF8String
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
|
||||
class DefaultSource extends TableProvider {
|
||||
override def inferSchema(options: CaseInsensitiveStringMap): StructType = OpenGaussTable.schema
|
||||
|
||||
override def getTable(
|
||||
schema: StructType,
|
||||
partitioning: Array[Transform],
|
||||
properties: util.Map[String, String]
|
||||
): Table = new OpenGaussTable(properties.get("tableName")) // TODO: Error handling
|
||||
}
|
||||
|
||||
class OpenGaussTable(val name: String) extends SupportsRead with SupportsWrite {
|
||||
override def schema(): StructType = OpenGaussTable.schema
|
||||
|
||||
override def capabilities(): util.Set[TableCapability] = Set(
|
||||
TableCapability.BATCH_READ,
|
||||
TableCapability.BATCH_WRITE
|
||||
).asJava
|
||||
|
||||
override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = new OpenGaussScanBuilder(options)
|
||||
|
||||
override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = new OpenGaussWriteBuilder(info.options)
|
||||
}
|
||||
|
||||
object OpenGaussTable {
|
||||
/*Table products*/
|
||||
/*Database school, table course*/
|
||||
val schema: StructType = new StructType().add("cor_id", IntegerType).add("cor_name", StringType).add("cor_type", StringType).add("credit", DoubleType)
|
||||
}
|
||||
|
||||
case class ConnectionProperties(url: String, user: String, password: String, tableName: String, partitionColumn: String, partitionSize: Int)
|
||||
|
||||
|
||||
|
||||
/** Read */
|
||||
|
||||
class OpenGaussScanBuilder(options: CaseInsensitiveStringMap) extends ScanBuilder {
|
||||
override def build(): Scan = new OpenGaussScan(ConnectionProperties(
|
||||
options.get("url"), options.get("user"), options.get("password"), options.get("tableName"), options.get("partitionColumn"), options.get("partitionSize").toInt
|
||||
))
|
||||
}
|
||||
|
||||
class OpenGaussPartition extends InputPartition
|
||||
|
||||
class OpenGaussScan(connectionProperties: ConnectionProperties) extends Scan with Batch {
|
||||
override def readSchema(): StructType = OpenGaussTable.schema
|
||||
|
||||
override def toBatch: Batch = this
|
||||
|
||||
override def planInputPartitions(): Array[InputPartition] = Array(new OpenGaussPartition)
|
||||
|
||||
override def createReaderFactory(): PartitionReaderFactory = new OpenGaussPartitionReaderFactory(connectionProperties)
|
||||
}
|
||||
|
||||
class OpenGaussPartitionReaderFactory(connectionProperties: ConnectionProperties)
|
||||
extends PartitionReaderFactory {
|
||||
override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new OpenGaussPartitionReader(connectionProperties)
|
||||
}
|
||||
|
||||
|
||||
|
||||
class OpenGaussPartitionReader(connectionProperties: ConnectionProperties) extends PartitionReader[InternalRow] {
|
||||
private val connection = DriverManager.getConnection(
|
||||
connectionProperties.url, connectionProperties.user, connectionProperties.password
|
||||
)
|
||||
private val statement = connection.createStatement()
|
||||
private val resultSet = statement.executeQuery(s"select * from ${connectionProperties.tableName}")
|
||||
|
||||
override def next(): Boolean = resultSet.next()
|
||||
|
||||
override def get(): InternalRow = InternalRow(
|
||||
resultSet.getInt(1),
|
||||
UTF8String.fromString(resultSet.getString(2)),
|
||||
UTF8String.fromString(resultSet.getString(3)),
|
||||
resultSet.getDouble(4))
|
||||
|
||||
override def close(): Unit = connection.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
/** Write */
|
||||
|
||||
class OpenGaussWriteBuilder(options: CaseInsensitiveStringMap) extends WriteBuilder {
|
||||
override def buildForBatch(): BatchWrite = new OpenGaussBatchWrite(ConnectionProperties(
|
||||
options.get("url"), options.get("user"), options.get("password"), options.get("tableName"), options.get("partitionColumn"), options.get("partitionSize").toInt
|
||||
))
|
||||
}
|
||||
|
||||
class OpenGaussBatchWrite(connectionProperties: ConnectionProperties) extends BatchWrite {
|
||||
override def createBatchWriterFactory(physicalWriteInfo: PhysicalWriteInfo): DataWriterFactory =
|
||||
new OpenGaussDataWriterFactory(connectionProperties)
|
||||
|
||||
override def commit(writerCommitMessages: Array[WriterCommitMessage]): Unit = {}
|
||||
|
||||
override def abort(writerCommitMessages: Array[WriterCommitMessage]): Unit = {}
|
||||
}
|
||||
|
||||
class OpenGaussDataWriterFactory(connectionProperties: ConnectionProperties) extends DataWriterFactory {
|
||||
override def createWriter(partitionId: Int, taskId:Long): DataWriter[InternalRow] =
|
||||
new OpenGaussWriter(connectionProperties)
|
||||
}
|
||||
|
||||
object WriteSucceeded extends WriterCommitMessage
|
||||
|
||||
class OpenGaussWriter(connectionProperties: ConnectionProperties) extends DataWriter[InternalRow] {
|
||||
|
||||
val connection = DriverManager.getConnection(
|
||||
connectionProperties.url,
|
||||
connectionProperties.user,
|
||||
connectionProperties.password
|
||||
)
|
||||
|
||||
// TODO：待修改
|
||||
val statement = "insert into ${connectionProperties.tableName} (cor_name, cor_type, credit) values (?,?,?)"
|
||||
val preparedStatement = connection.prepareStatement(statement)
|
||||
|
||||
override def write(record: InternalRow): Unit = {
|
||||
val cor_name = record.getString(0)
|
||||
val cor_type = record.getString(1)
|
||||
val credit = record.getDouble(2)
|
||||
|
||||
preparedStatement.setString(0, cor_name)
|
||||
preparedStatement.setString(1, cor_type)
|
||||
preparedStatement.setDouble(2, credit)
|
||||
preparedStatement.executeUpdate()
|
||||
}
|
||||
|
||||
override def commit(): WriterCommitMessage = WriteSucceeded
|
||||
|
||||
override def abort(): Unit = {}
|
||||
|
||||
override def close(): Unit = connection.close()
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
package org.opengauss.spark
|
||||
|
||||
import org.apache.spark.sql.{SaveMode, SparkSession}
|
||||
import org.scalatest.FlatSpec
|
||||
import java.sql.DriverManager
|
||||
import java.util.Properties
|
||||
|
||||
import org.scalatest.Matchers.convertToAnyShouldWrapper
|
||||
|
||||
class OpenGaussExample extends FlatSpec {
|
||||
|
||||
val testTableName = "course"
|
||||
|
||||
"Simple data source" should "read" in{
|
||||
val sparkSession = SparkSession.builder
|
||||
.master("local[2]")
|
||||
.appName("example")
|
||||
.getOrCreate()
|
||||
|
||||
val simpleDf = sparkSession.read
|
||||
.format("cn.ecnu.spark.sources.datasourcev2.simple")
|
||||
.load()
|
||||
|
||||
simpleDf.show()
|
||||
println(
|
||||
"number of partitions in simple source is " + simpleDf.rdd.getNumPartitions)
|
||||
}
|
||||
|
||||
|
||||
"openGauss data source" should "read table" in {
|
||||
val spark = SparkSession
|
||||
.builder()
|
||||
.master("local[*]")
|
||||
.appName("OpenGaussReaderJob")
|
||||
.getOrCreate()
|
||||
|
||||
val simpleRead = spark
|
||||
.read
|
||||
.format("org.opengauss.spark.sources.opengauss")
|
||||
.option("url", "jdbc:postgresql://x.x.x.x:port/school")
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "Enmo@123")
|
||||
.option("tableName", testTableName)
|
||||
.option("partitionSize", 10)
|
||||
// .option("partitionColumn", "name")
|
||||
.load()
|
||||
.show()
|
||||
|
||||
spark.stop()
|
||||
}
|
||||
|
||||
"openGauss data source" should "write table" in {
|
||||
val spark = SparkSession
|
||||
.builder()
|
||||
.master("local[*]")
|
||||
.appName("OpenGaussWriterJob")
|
||||
.getOrCreate()
|
||||
|
||||
import spark.implicits._
|
||||
|
||||
val df = (60 to 70).map(_.toLong).toDF("product_no")
|
||||
|
||||
df
|
||||
.write
|
||||
.format("org.opengauss.spark.sources.opengauss")
|
||||
.option("url", "jdbc:postgresql://x.x.x.x:port/postgres")
|
||||
.option("user", "sparkuser")
|
||||
.option("password", "Enmo@123")
|
||||
.option("tableName", testTableName)
|
||||
.option("partitionSize", 10)
|
||||
.option("partitionColumn", "product_no")
|
||||
.mode(SaveMode.Append)
|
||||
.save()
|
||||
|
||||
spark.stop()
|
||||
}
|
||||
|
||||
|
||||
|
||||
// def connection(c: PostgreSQLContainer) = {
|
||||
// Class.forName(c.driverClassName)
|
||||
// val properties = new Properties()
|
||||
// properties.put("user", c.username)
|
||||
// properties.put("password", c.password)
|
||||
// DriverManager.getConnection(c.jdbcUrl, properties)
|
||||
// }
|
||||
|
||||
object Queries {
|
||||
lazy val createTableQuery = s"CREATE TABLE $testTableName (user_id BIGINT PRIMARY KEY);"
|
||||
|
||||
lazy val testValues: String = (1 to 50).map(i => s"($i)").mkString(", ")
|
||||
|
||||
lazy val insertDataQuery = s"INSERT INTO $testTableName VALUES $testValues;"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
age,workclass,fnlwgt,education,education_num,martial_status,occupation,relationship,race,sex,capital_gain,capital_loss,hours_per_week,native_country,salary
|
||||
39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White, Male, 2174, 0, 40, United-States, <=50K
|
||||
50, Self-emp-not-inc, 83311, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 13, United-States, <=50K
|
||||
38, Private, 215646, HS-grad, 9, Divorced, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 234721, 11th, 7, Married-civ-spouse, Handlers-cleaners, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 338409, Bachelors, 13, Married-civ-spouse, Prof-specialty, Wife, Black, Female, 0, 0, 40, Cuba, <=50K
|
||||
37, Private, 284582, Masters, 14, Married-civ-spouse, Exec-managerial, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
49, Private, 160187, 9th, 5, Married-spouse-absent, Other-service, Not-in-family, Black, Female, 0, 0, 16, Jamaica, <=50K
|
||||
52, Self-emp-not-inc, 209642, HS-grad, 9, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
31, Private, 45781, Masters, 14, Never-married, Prof-specialty, Not-in-family, White, Female, 14084, 0, 50, United-States, >50K
|
||||
42, Private, 159449, Bachelors, 13, Married-civ-spouse, Exec-managerial, Husband, White, Male, 5178, 0, 40, United-States, >50K
|
||||
37, Private, 280464, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, Black, Male, 0, 0, 80, United-States, >50K
|
||||
30, State-gov, 141297, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Asian-Pac-Islander, Male, 0, 0, 40, India, >50K
|
||||
23, Private, 122272, Bachelors, 13, Never-married, Adm-clerical, Own-child, White, Female, 0, 0, 30, United-States, <=50K
|
||||
32, Private, 205019, Assoc-acdm, 12, Never-married, Sales, Not-in-family, Black, Male, 0, 0, 50, United-States, <=50K
|
||||
40, Private, 121772, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, Asian-Pac-Islander, Male, 0, 0, 40, ?, >50K
|
||||
34, Private, 245487, 7th-8th, 4, Married-civ-spouse, Transport-moving, Husband, Amer-Indian-Eskimo, Male, 0, 0, 45, Mexico, <=50K
|
||||
25, Self-emp-not-inc, 176756, HS-grad, 9, Never-married, Farming-fishing, Own-child, White, Male, 0, 0, 35, United-States, <=50K
|
||||
32, Private, 186824, HS-grad, 9, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, United-States, <=50K
|
||||
38, Private, 28887, 11th, 7, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
43, Self-emp-not-inc, 292175, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 45, United-States, >50K
|
||||
40, Private, 193524, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 60, United-States, >50K
|
||||
54, Private, 302146, HS-grad, 9, Separated, Other-service, Unmarried, Black, Female, 0, 0, 20, United-States, <=50K
|
||||
35, Federal-gov, 76845, 9th, 5, Married-civ-spouse, Farming-fishing, Husband, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Private, 117037, 11th, 7, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 2042, 40, United-States, <=50K
|
||||
59, Private, 109015, HS-grad, 9, Divorced, Tech-support, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
56, Local-gov, 216851, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
19, Private, 168294, HS-grad, 9, Never-married, Craft-repair, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
54, ?, 180211, Some-college, 10, Married-civ-spouse, ?, Husband, Asian-Pac-Islander, Male, 0, 0, 60, South, >50K
|
||||
39, Private, 367260, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 80, United-States, <=50K
|
||||
49, Private, 193366, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
23, Local-gov, 190709, Assoc-acdm, 12, Never-married, Protective-serv, Not-in-family, White, Male, 0, 0, 52, United-States, <=50K
|
||||
20, Private, 266015, Some-college, 10, Never-married, Sales, Own-child, Black, Male, 0, 0, 44, United-States, <=50K
|
||||
45, Private, 386940, Bachelors, 13, Divorced, Exec-managerial, Own-child, White, Male, 0, 1408, 40, United-States, <=50K
|
||||
30, Federal-gov, 59951, Some-college, 10, Married-civ-spouse, Adm-clerical, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
22, State-gov, 311512, Some-college, 10, Married-civ-spouse, Other-service, Husband, Black, Male, 0, 0, 15, United-States, <=50K
|
||||
48, Private, 242406, 11th, 7, Never-married, Machine-op-inspct, Unmarried, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
21, Private, 197200, Some-college, 10, Never-married, Machine-op-inspct, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 544091, HS-grad, 9, Married-AF-spouse, Adm-clerical, Wife, White, Female, 0, 0, 25, United-States, <=50K
|
||||
31, Private, 84154, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 38, ?, >50K
|
||||
48, Self-emp-not-inc, 265477, Assoc-acdm, 12, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
31, Private, 507875, 9th, 5, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 43, United-States, <=50K
|
||||
53, Self-emp-not-inc, 88506, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
24, Private, 172987, Bachelors, 13, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
49, Private, 94638, HS-grad, 9, Separated, Adm-clerical, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
25, Private, 289980, HS-grad, 9, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 35, United-States, <=50K
|
||||
57, Federal-gov, 337895, Bachelors, 13, Married-civ-spouse, Prof-specialty, Husband, Black, Male, 0, 0, 40, United-States, >50K
|
||||
53, Private, 144361, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 38, United-States, <=50K
|
||||
44, Private, 128354, Masters, 14, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
41, State-gov, 101603, Assoc-voc, 11, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
29, Private, 271466, Assoc-voc, 11, Never-married, Prof-specialty, Not-in-family, White, Male, 0, 0, 43, United-States, <=50K
|
||||
25, Private, 32275, Some-college, 10, Married-civ-spouse, Exec-managerial, Wife, Other, Female, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 226956, HS-grad, 9, Never-married, Other-service, Own-child, White, Female, 0, 0, 30, ?, <=50K
|
||||
47, Private, 51835, Prof-school, 15, Married-civ-spouse, Prof-specialty, Wife, White, Female, 0, 1902, 60, Honduras, >50K
|
||||
50, Federal-gov, 251585, Bachelors, 13, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 55, United-States, >50K
|
||||
47, Self-emp-inc, 109832, HS-grad, 9, Divorced, Exec-managerial, Not-in-family, White, Male, 0, 0, 60, United-States, <=50K
|
||||
43, Private, 237993, Some-college, 10, Married-civ-spouse, Tech-support, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
46, Private, 216666, 5th-6th, 3, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
35, Private, 56352, Assoc-voc, 11, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Puerto-Rico, <=50K
|
||||
41, Private, 147372, HS-grad, 9, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
30, Private, 188146, HS-grad, 9, Married-civ-spouse, Machine-op-inspct, Husband, White, Male, 5013, 0, 40, United-States, <=50K
|
||||
30, Private, 59496, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 2407, 0, 40, United-States, <=50K
|
||||
32, ?, 293936, 7th-8th, 4, Married-spouse-absent, ?, Not-in-family, White, Male, 0, 0, 40, ?, <=50K
|
||||
48, Private, 149640, HS-grad, 9, Married-civ-spouse, Transport-moving, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
42, Private, 116632, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 45, United-States, >50K
|
||||
29, Private, 105598, Some-college, 10, Divorced, Tech-support, Not-in-family, White, Male, 0, 0, 58, United-States, <=50K
|
||||
36, Private, 155537, HS-grad, 9, Married-civ-spouse, Craft-repair, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 183175, Some-college, 10, Divorced, Adm-clerical, Not-in-family, White, Female, 0, 0, 40, United-States, <=50K
|
||||
53, Private, 169846, HS-grad, 9, Married-civ-spouse, Adm-clerical, Wife, White, Female, 0, 0, 40, United-States, >50K
|
||||
49, Self-emp-inc, 191681, Some-college, 10, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 50, United-States, >50K
|
||||
25, ?, 200681, Some-college, 10, Never-married, ?, Own-child, White, Male, 0, 0, 40, United-States, <=50K
|
||||
19, Private, 101509, Some-college, 10, Never-married, Prof-specialty, Own-child, White, Male, 0, 0, 32, United-States, <=50K
|
||||
31, Private, 309974, Bachelors, 13, Separated, Sales, Own-child, Black, Female, 0, 0, 40, United-States, <=50K
|
||||
29, Self-emp-not-inc, 162298, Bachelors, 13, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 70, United-States, >50K
|
||||
23, Private, 211678, Some-college, 10, Never-married, Machine-op-inspct, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
79, Private, 124744, Some-college, 10, Married-civ-spouse, Prof-specialty, Other-relative, White, Male, 0, 0, 20, United-States, <=50K
|
||||
27, Private, 213921, HS-grad, 9, Never-married, Other-service, Own-child, White, Male, 0, 0, 40, Mexico, <=50K
|
||||
40, Private, 32214, Assoc-acdm, 12, Married-civ-spouse, Adm-clerical, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
67, ?, 212759, 10th, 6, Married-civ-spouse, ?, Husband, White, Male, 0, 0, 2, United-States, <=50K
|
||||
18, Private, 309634, 11th, 7, Never-married, Other-service, Own-child, White, Female, 0, 0, 22, United-States, <=50K
|
||||
31, Local-gov, 125927, 7th-8th, 4, Married-civ-spouse, Farming-fishing, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
18, Private, 446839, HS-grad, 9, Never-married, Sales, Not-in-family, White, Male, 0, 0, 30, United-States, <=50K
|
||||
52, Private, 276515, Bachelors, 13, Married-civ-spouse, Other-service, Husband, White, Male, 0, 0, 40, Cuba, <=50K
|
||||
46, Private, 51618, HS-grad, 9, Married-civ-spouse, Other-service, Wife, White, Female, 0, 0, 40, United-States, <=50K
|
||||
59, Private, 159937, HS-grad, 9, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, <=50K
|
||||
44, Private, 343591, HS-grad, 9, Divorced, Craft-repair, Not-in-family, White, Female, 14344, 0, 40, United-States, >50K
|
||||
53, Private, 346253, HS-grad, 9, Divorced, Sales, Own-child, White, Female, 0, 0, 35, United-States, <=50K
|
||||
49, Local-gov, 268234, HS-grad, 9, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
33, Private, 202051, Masters, 14, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 50, United-States, <=50K
|
||||
30, Private, 54334, 9th, 5, Never-married, Sales, Not-in-family, White, Male, 0, 0, 40, United-States, <=50K
|
||||
43, Federal-gov, 410867, Doctorate, 16, Never-married, Prof-specialty, Not-in-family, White, Female, 0, 0, 50, United-States, >50K
|
||||
57, Private, 249977, Assoc-voc, 11, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 0, 40, United-States, <=50K
|
||||
37, Private, 286730, Some-college, 10, Divorced, Craft-repair, Unmarried, White, Female, 0, 0, 40, United-States, <=50K
|
||||
28, Private, 212563, Some-college, 10, Divorced, Machine-op-inspct, Unmarried, Black, Female, 0, 0, 25, United-States, <=50K
|
||||
30, Private, 117747, HS-grad, 9, Married-civ-spouse, Sales, Wife, Asian-Pac-Islander, Female, 0, 1573, 35, ?, <=50K
|
||||
34, Local-gov, 226296, Bachelors, 13, Married-civ-spouse, Protective-serv, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
29, Local-gov, 115585, Some-college, 10, Never-married, Handlers-cleaners, Not-in-family, White, Male, 0, 0, 50, United-States, <=50K
|
||||
48, Self-emp-not-inc, 191277, Doctorate, 16, Married-civ-spouse, Prof-specialty, Husband, White, Male, 0, 1902, 60, United-States, >50K
|
||||
37, Private, 202683, Some-college, 10, Married-civ-spouse, Sales, Husband, White, Male, 0, 0, 48, United-States, >50K
|
||||
48, Private, 171095, Assoc-acdm, 12, Divorced, Exec-managerial, Unmarried, White, Female, 0, 0, 40, England, <=50K
|
||||
32, Federal-gov, 249409, HS-grad, 9, Never-married, Other-service, Own-child, Black, Male, 0, 0, 40, United-States, <=50K
|
||||
76, Private, 124191, Masters, 14, Married-civ-spouse, Exec-managerial, Husband, White, Male, 0, 0, 40, United-States, >50K
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
customerId,customerName
|
||||
1,John
|
||||
2,Clerk
|
||||
3,Micheal
|
||||
4,Sample
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
a||b||c||d
|
||||
1||2||3||4
|
||||
5||6||7||8
|
||||
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue