修改部分代码,添加部分注释以及删除无用注释与代码

Signed-off-by: Cerdore <khn64@163.com>
This commit is contained in:
Cerdore 2021-09-29 15:32:02 +08:00
parent 7010b4da86
commit 9964c996aa
3 changed files with 11 additions and 24 deletions

View File

@ -26,7 +26,7 @@ object SQLDataSourceExample {
// $example on:jdbc_dataset$
// Note: JDBC loading and saving can be achieved via either the load/save or jdbc methods
// Loading data from a JDBC source
val dburl = "jdbc:postgresql://x.x.x.x:port/school"
val dburl = "jdbc:postgresql://x.x.x.x:port/school" //注意修改此处的ip与端口地址
val jdbcDF = spark.read
.format("jdbc")
.option("url", dburl)
@ -65,6 +65,5 @@ object SQLDataSourceExample {
jdbcDF3.write
.option("createTableColumnTypes", "cla_id INT, cla_name VARCHAR(20)")
.jdbc(dburl, "customtable3", connectionProperties)
// $example off:jdbc_dataset$
}
}

View File

@ -127,18 +127,17 @@ class OpenGaussWriter(connectionProperties: ConnectionProperties) extends DataWr
connectionProperties.password
)
// TODO待修改
val statement = "insert into ${connectionProperties.tableName} (cor_name, cor_type, credit) values (?,?,?)"
val statement = "insert into ${connectionProperties.tableName} (cla_id, cla_name, cla_teacher) values (?,?,?)"
val preparedStatement = connection.prepareStatement(statement)
override def write(record: InternalRow): Unit = {
val cor_name = record.getString(0)
val cor_type = record.getString(1)
val credit = record.getDouble(2)
val cla_id = record.getInt(0)
val cla_name = record.getString(1)
val cla_teacher = record.getInt(2)
preparedStatement.setString(0, cor_name)
preparedStatement.setString(1, cor_type)
preparedStatement.setDouble(2, credit)
preparedStatement.setInt(0, cla_id)
preparedStatement.setString(1, cla_name)
preparedStatement.setInt(2, cla_teacher)
preparedStatement.executeUpdate()
}

View File

@ -10,7 +10,7 @@ import org.scalatest.Matchers.convertToAnyShouldWrapper
class OpenGaussExample extends FlatSpec {
val testTableName = "course"
val dburl = "jdbc:postgresql://x.x.x.x:port/school"
val dburl = "jdbc:postgresql://x.x.x.x:port/school" //注意将此处修改成你的机器对应的的ip与端口
"Simple data source" should "read" in{
val sparkSession = SparkSession.builder
@ -43,7 +43,6 @@ class OpenGaussExample extends FlatSpec {
.option("password", "pwdofsparkuser")
.option("tableName", testTableName)
.option("partitionSize", 10)
// .option("partitionColumn", "name")
.load()
.show()
@ -59,7 +58,7 @@ class OpenGaussExample extends FlatSpec {
import spark.implicits._
val df = (60 to 70).map(_.toLong).toDF("product_no")
val df = (60 to 70).map(_.toLong).toDF("cla_id")
df
.write
@ -69,22 +68,12 @@ class OpenGaussExample extends FlatSpec {
.option("password", "pwdofsparkuser")
.option("tableName", testTableName)
.option("partitionSize", 10)
.option("partitionColumn", "product_no")
.option("partitionColumn", "cla_id")
.mode(SaveMode.Append)
.save()
spark.stop()
}
object Queries {
lazy val createTableQuery = s"CREATE TABLE $testTableName (user_id BIGINT PRIMARY KEY);"
lazy val testValues: String = (1 to 50).map(i => s"($i)").mkString(", ")
lazy val insertDataQuery = s"INSERT INTO $testTableName VALUES $testValues;"
}
}